diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen')
103 files changed, 11181 insertions, 6488 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.cpp b/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.cpp index 1b56cf7c596d..edd7146dc1ac 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.cpp @@ -39,9 +39,9 @@ bool ABIInfo::isOHOSFamily() const { return getTarget().getTriple().isOHOSFamily(); } -Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { - return Address::invalid(); +RValue ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { + return RValue::getIgnored(); } bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { @@ -61,7 +61,7 @@ bool ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const { bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, uint64_t &Members) const { if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) { - uint64_t NElements = AT->getSize().getZExtValue(); + uint64_t NElements = AT->getZExtSize(); if (NElements == 0) return false; if (!isHomogeneousAggregate(AT->getElementType(), Base, Members)) @@ -98,7 +98,7 @@ bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, QualType FT = FD->getType(); while (const ConstantArrayType *AT = getContext().getAsConstantArrayType(FT)) { - if (AT->getSize().getZExtValue() == 0) + if (AT->isZeroSize()) return false; FT = AT->getElementType(); } @@ -184,6 +184,58 @@ ABIArgInfo ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, /*ByVal*/ false, Realign); } +void ABIInfo::appendAttributeMangling(TargetAttr *Attr, + raw_ostream &Out) const { + if (Attr->isDefaultVersion()) + return; + appendAttributeMangling(Attr->getFeaturesStr(), Out); +} + +void ABIInfo::appendAttributeMangling(TargetVersionAttr *Attr, + raw_ostream &Out) const { + appendAttributeMangling(Attr->getNamesStr(), Out); +} + +void ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index, + raw_ostream &Out) const { + appendAttributeMangling(Attr->getFeatureStr(Index), Out); + Out << '.' << Attr->getMangledIndex(Index); +} + +void ABIInfo::appendAttributeMangling(StringRef AttrStr, + raw_ostream &Out) const { + if (AttrStr == "default") { + Out << ".default"; + return; + } + + Out << '.'; + const TargetInfo &TI = CGT.getTarget(); + ParsedTargetAttr Info = TI.parseTargetAttr(AttrStr); + + llvm::sort(Info.Features, [&TI](StringRef LHS, StringRef RHS) { + // Multiversioning doesn't allow "no-${feature}", so we can + // only have "+" prefixes here. + assert(LHS.starts_with("+") && RHS.starts_with("+") && + "Features should always have a prefix."); + return TI.multiVersionSortPriority(LHS.substr(1)) > + TI.multiVersionSortPriority(RHS.substr(1)); + }); + + bool IsFirst = true; + if (!Info.CPU.empty()) { + IsFirst = false; + Out << "arch_" << Info.CPU; + } + + for (StringRef Feat : Info.Features) { + if (!IsFirst) + Out << '_'; + IsFirst = false; + Out << Feat.substr(1); + } +} + // Pin the vtable to this file. SwiftABIInfo::~SwiftABIInfo() = default; diff --git a/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.h b/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.h index b9a5ef6e4366..b8a8de57e5b9 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.h +++ b/contrib/llvm-project/clang/lib/CodeGen/ABIInfo.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_ABIINFO_H #define LLVM_CLANG_LIB_CODEGEN_ABIINFO_H +#include "clang/AST/Attr.h" #include "clang/AST/CharUnits.h" #include "clang/AST/Type.h" #include "llvm/IR/CallingConv.h" @@ -33,6 +34,8 @@ class CGCXXABI; class CGFunctionInfo; class CodeGenFunction; class CodeGenTypes; +class RValue; +class AggValueSlot; // FIXME: All of this stuff should be part of the target interface // somehow. It is currently here because it is not clear how to factor @@ -74,18 +77,18 @@ public: // the ABI information any lower than CodeGen. Of course, for // VAArg handling it has to be at this level; there is no way to // abstract this out. - virtual CodeGen::Address EmitVAArg(CodeGen::CodeGenFunction &CGF, - CodeGen::Address VAListAddr, - QualType Ty) const = 0; + virtual RValue EmitVAArg(CodeGen::CodeGenFunction &CGF, + CodeGen::Address VAListAddr, QualType Ty, + AggValueSlot Slot) const = 0; bool isAndroid() const; bool isOHOSFamily() const; /// Emit the target dependent code to load a value of /// \arg Ty from the \c __builtin_ms_va_list pointed to by \arg VAListAddr. - virtual CodeGen::Address EmitMSVAArg(CodeGen::CodeGenFunction &CGF, - CodeGen::Address VAListAddr, - QualType Ty) const; + virtual RValue EmitMSVAArg(CodeGen::CodeGenFunction &CGF, + CodeGen::Address VAListAddr, QualType Ty, + AggValueSlot Slot) const; virtual bool isHomogeneousAggregateBaseType(QualType Ty) const; @@ -111,6 +114,15 @@ public: CodeGen::ABIArgInfo getNaturalAlignIndirectInReg(QualType Ty, bool Realign = false) const; + + virtual void appendAttributeMangling(TargetAttr *Attr, + raw_ostream &Out) const; + virtual void appendAttributeMangling(TargetVersionAttr *Attr, + raw_ostream &Out) const; + virtual void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index, + raw_ostream &Out) const; + virtual void appendAttributeMangling(StringRef AttrStr, + raw_ostream &Out) const; }; /// Target specific hooks for defining how a type should be passed or returned diff --git a/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.cpp b/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.cpp index 2b20d5a13346..35e8f79ba1ba 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.cpp @@ -71,9 +71,12 @@ void DefaultABIInfo::computeInfo(CGFunctionInfo &FI) const { I.info = classifyArgumentType(I.type); } -Address DefaultABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { - return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty)); +RValue DefaultABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue( + EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty)), Ty), + Slot); } ABIArgInfo CodeGen::coerceToIntArray(QualType Ty, ASTContext &Context, @@ -157,7 +160,7 @@ llvm::Value *CodeGen::emitRoundPointerUpToAlignment(CodeGenFunction &CGF, llvm::Value *RoundUp = CGF.Builder.CreateConstInBoundsGEP1_32( CGF.Builder.getInt8Ty(), Ptr, Align.getQuantity() - 1); return CGF.Builder.CreateIntrinsic( - llvm::Intrinsic::ptrmask, {CGF.AllocaInt8PtrTy, CGF.IntPtrTy}, + llvm::Intrinsic::ptrmask, {Ptr->getType(), CGF.IntPtrTy}, {RoundUp, llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity())}, nullptr, Ptr->getName() + ".aligned"); } @@ -187,7 +190,7 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr, CharUnits FullDirectSize = DirectSize.alignTo(SlotSize); Address NextPtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next"); - CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr); + CGF.Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); // If the argument is smaller than a slot, and this is a big-endian // target, the argument will be right-adjusted in its slot. @@ -199,12 +202,12 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr, return Addr.withElementType(DirectTy); } -Address CodeGen::emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType ValueTy, bool IsIndirect, - TypeInfoChars ValueInfo, - CharUnits SlotSizeAndAlign, - bool AllowHigherAlign, - bool ForceRightAdjust) { +RValue CodeGen::emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType ValueTy, bool IsIndirect, + TypeInfoChars ValueInfo, + CharUnits SlotSizeAndAlign, + bool AllowHigherAlign, AggValueSlot Slot, + bool ForceRightAdjust) { // The size and alignment of the value that was passed directly. CharUnits DirectSize, DirectAlign; if (IsIndirect) { @@ -230,7 +233,7 @@ Address CodeGen::emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, Addr = Address(CGF.Builder.CreateLoad(Addr), ElementTy, ValueInfo.Align); } - return Addr; + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(Addr, ValueTy), Slot); } Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1, @@ -239,15 +242,15 @@ Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1, const llvm::Twine &Name) { assert(Addr1.getType() == Addr2.getType()); llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name); - PHI->addIncoming(Addr1.getPointer(), Block1); - PHI->addIncoming(Addr2.getPointer(), Block2); + PHI->addIncoming(Addr1.emitRawPointer(CGF), Block1); + PHI->addIncoming(Addr2.emitRawPointer(CGF), Block2); CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment()); return Address(PHI, Addr1.getElementType(), Align); } bool CodeGen::isEmptyField(ASTContext &Context, const FieldDecl *FD, bool AllowArrays, bool AsIfNoUniqueAddr) { - if (FD->isUnnamedBitfield()) + if (FD->isUnnamedBitField()) return true; QualType FT = FD->getType(); @@ -257,7 +260,7 @@ bool CodeGen::isEmptyField(ASTContext &Context, const FieldDecl *FD, bool WasArray = false; if (AllowArrays) while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) { - if (AT->getSize() == 0) + if (AT->isZeroSize()) return true; FT = AT->getElementType(); // The [[no_unique_address]] special case below does not apply to @@ -307,6 +310,41 @@ bool CodeGen::isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays, return true; } +bool CodeGen::isEmptyFieldForLayout(const ASTContext &Context, + const FieldDecl *FD) { + if (FD->isZeroLengthBitField(Context)) + return true; + + if (FD->isUnnamedBitField()) + return false; + + return isEmptyRecordForLayout(Context, FD->getType()); +} + +bool CodeGen::isEmptyRecordForLayout(const ASTContext &Context, QualType T) { + const RecordType *RT = T->getAs<RecordType>(); + if (!RT) + return false; + + const RecordDecl *RD = RT->getDecl(); + + // If this is a C++ record, check the bases first. + if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { + if (CXXRD->isDynamicClass()) + return false; + + for (const auto &I : CXXRD->bases()) + if (!isEmptyRecordForLayout(Context, I.getType())) + return false; + } + + for (const auto *I : RD->fields()) + if (!isEmptyFieldForLayout(Context, I)) + return false; + + return true; +} + const Type *CodeGen::isSingleElementStruct(QualType T, ASTContext &Context) { const RecordType *RT = T->getAs<RecordType>(); if (!RT) @@ -352,7 +390,7 @@ const Type *CodeGen::isSingleElementStruct(QualType T, ASTContext &Context) { // Treat single element arrays as the element. while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) { - if (AT->getSize().getZExtValue() != 1) + if (AT->getZExtSize() != 1) break; FT = AT->getElementType(); } @@ -400,7 +438,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty); llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy); llvm::Value *Addr = - CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy); + CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), BaseTy); return Address(Addr, ElementTy, TyAlignForABI); } else { assert((AI.isDirect() || AI.isExtend()) && @@ -416,7 +454,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!"); Address Temp = CGF.CreateMemTemp(Ty, "varet"); - Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), + Val = CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Ty)); CGF.Builder.CreateStore(Val, Temp); return Temp; diff --git a/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.h b/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.h index afde08ba100c..2a3ef6b8a6c9 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.h +++ b/contrib/llvm-project/clang/lib/CodeGen/ABIInfoImpl.h @@ -29,8 +29,8 @@ public: void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; // Helper for coercing an aggregate argument or return value into an integer @@ -112,10 +112,11 @@ Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr, /// \param ForceRightAdjust - Default is false. On big-endian platform and /// if the argument is smaller than a slot, set this flag will force /// right-adjust the argument in its slot irrespective of the type. -Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType ValueTy, bool IsIndirect, - TypeInfoChars ValueInfo, CharUnits SlotSizeAndAlign, - bool AllowHigherAlign, bool ForceRightAdjust = false); +RValue emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType ValueTy, bool IsIndirect, + TypeInfoChars ValueInfo, CharUnits SlotSizeAndAlign, + bool AllowHigherAlign, AggValueSlot Slot, + bool ForceRightAdjust = false); Address emitMergePHI(CodeGenFunction &CGF, Address Addr1, llvm::BasicBlock *Block1, Address Addr2, @@ -136,6 +137,16 @@ bool isEmptyField(ASTContext &Context, const FieldDecl *FD, bool AllowArrays, bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays, bool AsIfNoUniqueAddr = false); +/// isEmptyFieldForLayout - Return true iff the field is "empty", that is, +/// either a zero-width bit-field or an \ref isEmptyRecordForLayout. +bool isEmptyFieldForLayout(const ASTContext &Context, const FieldDecl *FD); + +/// isEmptyRecordForLayout - Return true iff a structure contains only empty +/// base classes (per \ref isEmptyRecordForLayout) and fields (per +/// \ref isEmptyFieldForLayout). Note, C++ record fields are considered empty +/// if the [[no_unique_address]] attribute would have made them empty. +bool isEmptyRecordForLayout(const ASTContext &Context, QualType T); + /// isSingleElementStruct - Determine if a structure is a "single /// element struct", i.e. it has exactly one non-empty field or /// exactly one field which is itself a single element diff --git a/contrib/llvm-project/clang/lib/CodeGen/Address.h b/contrib/llvm-project/clang/lib/CodeGen/Address.h index cf48df8f5e73..a18c7169af1e 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Address.h +++ b/contrib/llvm-project/clang/lib/CodeGen/Address.h @@ -14,7 +14,9 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_ADDRESS_H #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H +#include "CGPointerAuthInfo.h" #include "clang/AST/CharUnits.h" +#include "clang/AST/Type.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/IR/Constants.h" #include "llvm/Support/MathExtras.h" @@ -22,28 +24,41 @@ namespace clang { namespace CodeGen { +class Address; +class CGBuilderTy; +class CodeGenFunction; +class CodeGenModule; + // Indicates whether a pointer is known not to be null. enum KnownNonNull_t { NotKnownNonNull, KnownNonNull }; -/// An aligned address. -class Address { +/// An abstract representation of an aligned address. This is designed to be an +/// IR-level abstraction, carrying just the information necessary to perform IR +/// operations on an address like loads and stores. In particular, it doesn't +/// carry C type information or allow the representation of things like +/// bit-fields; clients working at that level should generally be using +/// `LValue`. +/// The pointer contained in this class is known to be unsigned. +class RawAddress { llvm::PointerIntPair<llvm::Value *, 1, bool> PointerAndKnownNonNull; llvm::Type *ElementType; CharUnits Alignment; protected: - Address(std::nullptr_t) : ElementType(nullptr) {} + RawAddress(std::nullptr_t) : ElementType(nullptr) {} public: - Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, - KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + RawAddress(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) : PointerAndKnownNonNull(Pointer, IsKnownNonNull), ElementType(ElementType), Alignment(Alignment) { assert(Pointer != nullptr && "Pointer cannot be null"); assert(ElementType != nullptr && "Element type cannot be null"); } - static Address invalid() { return Address(nullptr); } + inline RawAddress(Address Addr); + + static RawAddress invalid() { return RawAddress(nullptr); } bool isValid() const { return PointerAndKnownNonNull.getPointer() != nullptr; } @@ -80,6 +95,165 @@ public: return Alignment; } + /// Return address with different element type, but same pointer and + /// alignment. + RawAddress withElementType(llvm::Type *ElemTy) const { + return RawAddress(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); + } + + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); + } +}; + +/// Like RawAddress, an abstract representation of an aligned address, but the +/// pointer contained in this class is possibly signed. +/// +/// This is designed to be an IR-level abstraction, carrying just the +/// information necessary to perform IR operations on an address like loads and +/// stores. In particular, it doesn't carry C type information or allow the +/// representation of things like bit-fields; clients working at that level +/// should generally be using `LValue`. +/// +/// An address may be either *raw*, meaning that it's an ordinary machine +/// pointer, or *signed*, meaning that the pointer carries an embedded +/// pointer-authentication signature. Representing signed pointers directly in +/// this abstraction allows the authentication to be delayed as long as possible +/// without forcing IRGen to use totally different code paths for signed and +/// unsigned values or to separately propagate signature information through +/// every API that manipulates addresses. Pointer arithmetic on signed addresses +/// (e.g. drilling down to a struct field) is accumulated into a separate offset +/// which is applied when the address is finally accessed. +class Address { + friend class CGBuilderTy; + + // The boolean flag indicates whether the pointer is known to be non-null. + llvm::PointerIntPair<llvm::Value *, 1, bool> Pointer; + + /// The expected IR type of the pointer. Carrying accurate element type + /// information in Address makes it more convenient to work with Address + /// values and allows frontend assertions to catch simple mistakes. + llvm::Type *ElementType = nullptr; + + CharUnits Alignment; + + /// The ptrauth information needed to authenticate the base pointer. + CGPointerAuthInfo PtrAuthInfo; + + /// Offset from the base pointer. This is non-null only when the base + /// pointer is signed. + llvm::Value *Offset = nullptr; + + llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const; + +protected: + Address(std::nullptr_t) : ElementType(nullptr) {} + +public: + Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : Pointer(pointer, IsKnownNonNull), ElementType(elementType), + Alignment(alignment) { + assert(pointer != nullptr && "Pointer cannot be null"); + assert(elementType != nullptr && "Element type cannot be null"); + assert(!alignment.isZero() && "Alignment cannot be zero"); + } + + Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment, + CGPointerAuthInfo PtrAuthInfo, llvm::Value *Offset, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType), + Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset) {} + + Address(RawAddress RawAddr) + : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr, + RawAddr.isValid() ? RawAddr.isKnownNonNull() : NotKnownNonNull), + ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr), + Alignment(RawAddr.isValid() ? RawAddr.getAlignment() + : CharUnits::Zero()) {} + + static Address invalid() { return Address(nullptr); } + bool isValid() const { return Pointer.getPointer() != nullptr; } + + /// This function is used in situations where the caller is doing some sort of + /// opaque "laundering" of the pointer. + void replaceBasePointer(llvm::Value *P) { + assert(isValid() && "pointer isn't valid"); + assert(P->getType() == Pointer.getPointer()->getType() && + "Pointer's type changed"); + Pointer.setPointer(P); + assert(isValid() && "pointer is invalid after replacement"); + } + + CharUnits getAlignment() const { return Alignment; } + + void setAlignment(CharUnits Value) { Alignment = Value; } + + llvm::Value *getBasePointer() const { + assert(isValid() && "pointer isn't valid"); + return Pointer.getPointer(); + } + + /// Return the type of the pointer value. + llvm::PointerType *getType() const { + return llvm::PointerType::get( + ElementType, + llvm::cast<llvm::PointerType>(Pointer.getPointer()->getType()) + ->getAddressSpace()); + } + + /// Return the type of the values stored in this address. + llvm::Type *getElementType() const { + assert(isValid()); + return ElementType; + } + + /// Return the address space that this address resides in. + unsigned getAddressSpace() const { return getType()->getAddressSpace(); } + + /// Return the IR name of the pointer value. + llvm::StringRef getName() const { return Pointer.getPointer()->getName(); } + + const CGPointerAuthInfo &getPointerAuthInfo() const { return PtrAuthInfo; } + void setPointerAuthInfo(const CGPointerAuthInfo &Info) { PtrAuthInfo = Info; } + + // This function is called only in CGBuilderBaseTy::CreateElementBitCast. + void setElementType(llvm::Type *Ty) { + assert(hasOffset() && + "this funcion shouldn't be called when there is no offset"); + ElementType = Ty; + } + + bool isSigned() const { return PtrAuthInfo.isSigned(); } + + /// Whether the pointer is known not to be null. + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)Pointer.getInt(); + } + + Address setKnownNonNull() { + assert(isValid()); + Pointer.setInt(KnownNonNull); + return *this; + } + + bool hasOffset() const { return Offset; } + + llvm::Value *getOffset() const { return Offset; } + + Address getResignedAddress(const CGPointerAuthInfo &NewInfo, + CodeGenFunction &CGF) const; + + /// Return the pointer contained in this class after authenticating it and + /// adding offset to it if necessary. + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { + if (!isSigned()) + return getBasePointer(); + return emitRawPointerSlow(CGF); + } + /// Return address with different pointer, but same element type and /// alignment. Address withPointer(llvm::Value *NewPointer, @@ -91,61 +265,60 @@ public: /// Return address with different alignment, but same pointer and element /// type. Address withAlignment(CharUnits NewAlignment) const { - return Address(getPointer(), getElementType(), NewAlignment, + return Address(Pointer.getPointer(), getElementType(), NewAlignment, isKnownNonNull()); } /// Return address with different element type, but same pointer and /// alignment. Address withElementType(llvm::Type *ElemTy) const { - return Address(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); - } - - /// Whether the pointer is known not to be null. - KnownNonNull_t isKnownNonNull() const { - assert(isValid()); - return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); - } - - /// Set the non-null bit. - Address setKnownNonNull() { - assert(isValid()); - PointerAndKnownNonNull.setInt(true); - return *this; + if (!hasOffset()) + return Address(getBasePointer(), ElemTy, getAlignment(), + getPointerAuthInfo(), /*Offset=*/nullptr, + isKnownNonNull()); + Address A(*this); + A.ElementType = ElemTy; + return A; } }; +inline RawAddress::RawAddress(Address Addr) + : PointerAndKnownNonNull(Addr.isValid() ? Addr.getBasePointer() : nullptr, + Addr.isValid() ? Addr.isKnownNonNull() + : NotKnownNonNull), + ElementType(Addr.isValid() ? Addr.getElementType() : nullptr), + Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {} + /// A specialization of Address that requires the address to be an /// LLVM Constant. -class ConstantAddress : public Address { - ConstantAddress(std::nullptr_t) : Address(nullptr) {} +class ConstantAddress : public RawAddress { + ConstantAddress(std::nullptr_t) : RawAddress(nullptr) {} public: ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType, CharUnits alignment) - : Address(pointer, elementType, alignment) {} + : RawAddress(pointer, elementType, alignment) {} static ConstantAddress invalid() { return ConstantAddress(nullptr); } llvm::Constant *getPointer() const { - return llvm::cast<llvm::Constant>(Address::getPointer()); + return llvm::cast<llvm::Constant>(RawAddress::getPointer()); } ConstantAddress withElementType(llvm::Type *ElemTy) const { return ConstantAddress(getPointer(), ElemTy, getAlignment()); } - static bool isaImpl(Address addr) { + static bool isaImpl(RawAddress addr) { return llvm::isa<llvm::Constant>(addr.getPointer()); } - static ConstantAddress castImpl(Address addr) { + static ConstantAddress castImpl(RawAddress addr) { return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()), addr.getElementType(), addr.getAlignment()); } }; - } // Present a minimal LLVM-like casting interface. diff --git a/contrib/llvm-project/clang/lib/CodeGen/BackendConsumer.h b/contrib/llvm-project/clang/lib/CodeGen/BackendConsumer.h index 72a814cd43d7..a023d29cbd1d 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/BackendConsumer.h +++ b/contrib/llvm-project/clang/lib/CodeGen/BackendConsumer.h @@ -74,8 +74,8 @@ public: const HeaderSearchOptions &HeaderSearchOpts, const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts, - const TargetOptions &TargetOpts, - const LangOptions &LangOpts, const std::string &InFile, + const TargetOptions &TargetOpts, const LangOptions &LangOpts, + const std::string &InFile, SmallVector<LinkModule, 4> LinkModules, std::unique_ptr<raw_pwrite_stream> OS, llvm::LLVMContext &C, CoverageSourceInfo *CoverageInfo = nullptr); @@ -88,9 +88,9 @@ public: const HeaderSearchOptions &HeaderSearchOpts, const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts, - const TargetOptions &TargetOpts, - const LangOptions &LangOpts, llvm::Module *Module, - SmallVector<LinkModule, 4> LinkModules, llvm::LLVMContext &C, + const TargetOptions &TargetOpts, const LangOptions &LangOpts, + llvm::Module *Module, SmallVector<LinkModule, 4> LinkModules, + llvm::LLVMContext &C, CoverageSourceInfo *CoverageInfo = nullptr); llvm::Module *getModule() const; @@ -107,13 +107,12 @@ public: void HandleTagDeclDefinition(TagDecl *D) override; void HandleTagDeclRequiredDefinition(const TagDecl *D) override; void CompleteTentativeDefinition(VarDecl *D) override; - void CompleteExternalDeclaration(VarDecl *D) override; + void CompleteExternalDeclaration(DeclaratorDecl *D) override; void AssignInheritanceModel(CXXRecordDecl *RD) override; void HandleVTable(CXXRecordDecl *RD) override; - - // Links each entry in LinkModules into our module. Returns true on error. - bool LinkInModules(llvm::Module *M, bool ShouldLinkFiles = true); + // Links each entry in LinkModules into our module. Returns true on error. + bool LinkInModules(llvm::Module *M); /// Get the best possible source location to represent a diagnostic that /// may have associated debug info. diff --git a/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp index 4f22d35f9d3a..e765bbf637a6 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp @@ -73,8 +73,10 @@ #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/Transforms/Instrumentation/KCFI.h" +#include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h" #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" @@ -84,7 +86,6 @@ #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Scalar/JumpThreading.h" #include "llvm/Transforms/Utils/Debugify.h" -#include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include <memory> #include <optional> @@ -101,14 +102,24 @@ extern cl::opt<bool> PrintPipelinePasses; // Experiment to move sanitizers earlier. static cl::opt<bool> ClSanitizeOnOptimizerEarlyEP( "sanitizer-early-opt-ep", cl::Optional, - cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); + cl::desc("Insert sanitizers on OptimizerEarlyEP.")); + +// Experiment to mark cold functions as optsize/minsize/optnone. +// TODO: remove once this is exposed as a proper driver flag. +static cl::opt<PGOOptions::ColdFuncOpt> ClPGOColdFuncAttr( + "pgo-cold-func-opt", cl::init(PGOOptions::ColdFuncOpt::Default), cl::Hidden, + cl::desc( + "Function attribute to apply to cold functions as determined by PGO"), + cl::values(clEnumValN(PGOOptions::ColdFuncOpt::Default, "default", + "Default (no attribute)"), + clEnumValN(PGOOptions::ColdFuncOpt::OptSize, "optsize", + "Mark cold functions with optsize."), + clEnumValN(PGOOptions::ColdFuncOpt::MinSize, "minsize", + "Mark cold functions with minsize."), + clEnumValN(PGOOptions::ColdFuncOpt::OptNone, "optnone", + "Mark cold functions with optnone."))); extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate; - -// Re-link builtin bitcodes after optimization -cl::opt<bool> ClRelinkBuiltinBitcodePostop( - "relink-builtin-bitcode-postop", cl::Optional, - cl::desc("Re-link builtin bitcodes after optimization."), cl::init(false)); } // namespace llvm namespace { @@ -356,8 +367,6 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, llvm::TargetMachine::parseBinutilsVersion(CodeGenOpts.BinutilsVersion); Options.UseInitArray = CodeGenOpts.UseInitArray; Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS; - Options.CompressDebugSections = CodeGenOpts.getCompressDebugSections(); - Options.RelaxELFRelocations = CodeGenOpts.RelaxELFRelocations; // Set EABI version. Options.EABIVersion = TargetOpts.EABIVersion; @@ -382,6 +391,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, LangOptions::FPModeKind::FPM_FastHonorPragmas); Options.ApproxFuncFPMath = LangOpts.ApproxFunc; + Options.BBAddrMap = CodeGenOpts.BBAddrMap; Options.BBSections = llvm::StringSwitch<llvm::BasicBlockSection>(CodeGenOpts.BBSections) .Case("all", llvm::BasicBlockSection::All) @@ -408,6 +418,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames; Options.UniqueBasicBlockSectionNames = CodeGenOpts.UniqueBasicBlockSectionNames; + Options.SeparateNamedSections = CodeGenOpts.SeparateNamedSections; Options.TLSSize = CodeGenOpts.TLSSize; Options.EnableTLSDESC = CodeGenOpts.EnableTLSDESC; Options.EmulatedTLS = CodeGenOpts.EmulatedTLS; @@ -459,6 +470,10 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose; Options.MCOptions.Dwarf64 = CodeGenOpts.Dwarf64; Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments; + Options.MCOptions.Crel = CodeGenOpts.Crel; + Options.MCOptions.X86RelaxRelocations = CodeGenOpts.RelaxELFRelocations; + Options.MCOptions.CompressDebugSections = + CodeGenOpts.getCompressDebugSections(); Options.MCOptions.ABIName = TargetOpts.ABI; for (const auto &Entry : HSOpts.UserEntries) if (!Entry.IsFramework && @@ -694,6 +709,9 @@ static void addSanitizers(const Triple &TargetTriple, MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); } + if (LangOpts.Sanitize.has(SanitizerKind::NumericalStability)) + MPM.addPass(NumericalStabilitySanitizerPass()); + auto ASanPass = [&](SanitizerMask Mask, bool CompileKernel) { if (LangOpts.Sanitize.has(Mask)) { bool UseGlobalGC = asanUseGlobalsGC(TargetTriple, CodeGenOpts); @@ -742,6 +760,16 @@ static void addSanitizers(const Triple &TargetTriple, // LastEP does not need GlobalsAA. PB.registerOptimizerLastEPCallback(SanitizersCallback); } + + if (LowerAllowCheckPass::IsRequested()) { + // We can optimize after inliner, and PGO profile matching. The hook below + // is called at the end `buildFunctionSimplificationPipeline`, which called + // from `buildInlinerPipeline`, which called after profile matching. + PB.registerScalarOptimizerLateEPCallback( + [](FunctionPassManager &FPM, OptimizationLevel Level) { + FPM.addPass(LowerAllowCheckPass()); + }); + } } void EmitAssemblyHelper::RunOptimizationPipeline( @@ -755,37 +783,41 @@ void EmitAssemblyHelper::RunOptimizationPipeline( CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput, "", "", CodeGenOpts.MemoryProfileUsePath, nullptr, PGOOptions::IRInstr, - PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling, + PGOOptions::NoCSAction, ClPGOColdFuncAttr, + CodeGenOpts.DebugInfoForProfiling, /*PseudoProbeForProfiling=*/false, CodeGenOpts.AtomicProfileUpdate); else if (CodeGenOpts.hasProfileIRUse()) { // -fprofile-use. auto CSAction = CodeGenOpts.hasProfileCSIRUse() ? PGOOptions::CSIRUse : PGOOptions::NoCSAction; - PGOOpt = PGOOptions( - CodeGenOpts.ProfileInstrumentUsePath, "", - CodeGenOpts.ProfileRemappingFile, CodeGenOpts.MemoryProfileUsePath, VFS, - PGOOptions::IRUse, CSAction, CodeGenOpts.DebugInfoForProfiling); + PGOOpt = PGOOptions(CodeGenOpts.ProfileInstrumentUsePath, "", + CodeGenOpts.ProfileRemappingFile, + CodeGenOpts.MemoryProfileUsePath, VFS, + PGOOptions::IRUse, CSAction, ClPGOColdFuncAttr, + CodeGenOpts.DebugInfoForProfiling); } else if (!CodeGenOpts.SampleProfileFile.empty()) // -fprofile-sample-use PGOOpt = PGOOptions( CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile, CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse, - PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling, - CodeGenOpts.PseudoProbeForProfiling); + PGOOptions::NoCSAction, ClPGOColdFuncAttr, + CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling); else if (!CodeGenOpts.MemoryProfileUsePath.empty()) // -fmemory-profile-use (without any of the above options) PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::NoAction, PGOOptions::NoCSAction, - CodeGenOpts.DebugInfoForProfiling); + ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.PseudoProbeForProfiling) // -fpseudo-probe-for-profiling - PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr, - PGOOptions::NoAction, PGOOptions::NoCSAction, - CodeGenOpts.DebugInfoForProfiling, true); + PGOOpt = + PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr, + PGOOptions::NoAction, PGOOptions::NoCSAction, + ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling, true); else if (CodeGenOpts.DebugInfoForProfiling) // -fdebug-info-for-profiling PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr, - PGOOptions::NoAction, PGOOptions::NoCSAction, true); + PGOOptions::NoAction, PGOOptions::NoCSAction, + ClPGOColdFuncAttr, true); // Check to see if we want to generate a CS profile. if (CodeGenOpts.hasProfileCSIRInstr()) { @@ -802,13 +834,13 @@ void EmitAssemblyHelper::RunOptimizationPipeline( : CodeGenOpts.InstrProfileOutput; PGOOpt->CSAction = PGOOptions::CSIRInstr; } else - PGOOpt = - PGOOptions("", - CodeGenOpts.InstrProfileOutput.empty() - ? getDefaultProfileGenName() - : CodeGenOpts.InstrProfileOutput, - "", /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction, - PGOOptions::CSIRInstr, CodeGenOpts.DebugInfoForProfiling); + PGOOpt = PGOOptions("", + CodeGenOpts.InstrProfileOutput.empty() + ? getDefaultProfileGenName() + : CodeGenOpts.InstrProfileOutput, + "", /*MemoryProfile=*/"", nullptr, + PGOOptions::NoAction, PGOOptions::CSIRInstr, + ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling); } if (TM) TM->setPGOOption(PGOOpt); @@ -955,22 +987,6 @@ void EmitAssemblyHelper::RunOptimizationPipeline( /*DropTypeTests=*/true)); }); - if (CodeGenOpts.InstrumentFunctions || - CodeGenOpts.InstrumentFunctionEntryBare || - CodeGenOpts.InstrumentFunctionsAfterInlining || - CodeGenOpts.InstrumentForProfiling) { - PB.registerPipelineStartEPCallback( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(createModuleToFunctionPassAdaptor( - EntryExitInstrumenterPass(/*PostInlining=*/false))); - }); - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(createModuleToFunctionPassAdaptor( - EntryExitInstrumenterPass(/*PostInlining=*/true))); - }); - } - // Register callbacks to schedule sanitizer passes at the appropriate part // of the pipeline. if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) @@ -1022,12 +1038,9 @@ void EmitAssemblyHelper::RunOptimizationPipeline( } } - // Re-link against any bitcodes supplied via the -mlink-builtin-bitcode option - // Some optimizations may generate new function calls that would not have - // been linked pre-optimization (i.e. fused sincos calls generated by - // AMDGPULibCalls::fold_sincos.) - if (ClRelinkBuiltinBitcodePostop) - MPM.addPass(LinkInModulesPass(BC, false)); + // Link against bitcodes supplied via the -mlink-builtin-bitcode option + if (CodeGenOpts.LinkBitcodePostopt) + MPM.addPass(LinkInModulesPass(BC)); // Add a verifier pass if requested. We don't have to do this if the action // requires code generation because there will already be a verifier pass in diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp index 52e6ddb7d6af..fbf942d06ca6 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp @@ -80,7 +80,7 @@ namespace { AtomicSizeInBits = C.toBits( C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) .alignTo(lvalue.getAlignment())); - llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer(); + llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); auto OffsetInChars = (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * lvalue.getAlignment(); @@ -139,18 +139,18 @@ namespace { const LValue &getAtomicLValue() const { return LVal; } llvm::Value *getAtomicPointer() const { if (LVal.isSimple()) - return LVal.getPointer(CGF); + return LVal.emitRawPointer(CGF); else if (LVal.isBitField()) - return LVal.getBitFieldPointer(); + return LVal.getRawBitFieldPointer(CGF); else if (LVal.isVectorElt()) - return LVal.getVectorPointer(); + return LVal.getRawVectorPointer(CGF); assert(LVal.isExtVectorElt()); - return LVal.getExtVectorPointer(); + return LVal.getRawExtVectorPointer(CGF); } Address getAtomicAddress() const { llvm::Type *ElTy; if (LVal.isSimple()) - ElTy = LVal.getAddress(CGF).getElementType(); + ElTy = LVal.getAddress().getElementType(); else if (LVal.isBitField()) ElTy = LVal.getBitFieldAddress().getElementType(); else if (LVal.isVectorElt()) @@ -194,12 +194,14 @@ namespace { RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, SourceLocation loc, bool AsValue) const; - /// Converts a rvalue to integer value. - llvm::Value *convertRValueToInt(RValue RVal) const; + llvm::Value *getScalarRValValueOrNull(RValue RVal) const; - RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, - AggValueSlot ResultSlot, - SourceLocation Loc, bool AsValue) const; + /// Converts an rvalue to integer value if needed. + llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const; + + RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot, + SourceLocation Loc, bool AsValue, + bool CmpXchg = false) const; /// Copy an atomic r-value into atomic-layout memory. void emitCopyIntoMemory(RValue rvalue) const; @@ -261,7 +263,8 @@ namespace { void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, llvm::AtomicOrdering AO, bool IsVolatile); /// Emits atomic load as LLVM instruction. - llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); + llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile, + bool CmpXchg = false); /// Emits atomic compare-and-exchange op as a libcall. llvm::Value *EmitAtomicCompareExchangeLibcall( llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, @@ -360,12 +363,12 @@ bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { bool AtomicInfo::emitMemSetZeroIfNecessary() const { assert(LVal.isSimple()); - Address addr = LVal.getAddress(CGF); + Address addr = LVal.getAddress(); if (!requiresMemSetZero(addr.getElementType())) return false; CGF.Builder.CreateMemSet( - addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), + addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0), CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), LVal.getAlignment().getAsAlign()); return true; @@ -811,29 +814,6 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, Builder.SetInsertPoint(ContBB); } -static void -AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, - bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, - SourceLocation Loc, CharUnits SizeInChars) { - if (UseOptimizedLibcall) { - // Load value and pass it to the function directly. - CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); - int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); - ValTy = - CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); - llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits); - Address Ptr = Address(Val, ITy, Align); - Val = CGF.EmitLoadOfScalar(Ptr, false, - CGF.getContext().getPointerType(ValTy), - Loc); - // Coerce the value into an appropriately sized integer type. - Args.add(RValue::get(Val), ValTy); - } else { - // Non-optimized functions always take a reference. - Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy); - } -} - RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); QualType MemTy = AtomicTy; @@ -857,22 +837,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { uint64_t Size = TInfo.Width.getQuantity(); unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); - bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; - bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; - bool UseLibcall = Misaligned | Oversized; - bool ShouldCastToIntPtrTy = true; - CharUnits MaxInlineWidth = getContext().toCharUnitsFromBits(MaxInlineWidthInBits); - DiagnosticsEngine &Diags = CGM.getDiags(); - + bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; + bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; if (Misaligned) { Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) << (int)TInfo.Width.getQuantity() << (int)Ptr.getAlignment().getQuantity(); } - if (Oversized) { Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); @@ -881,6 +855,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm::Value *Order = EmitScalarExpr(E->getOrder()); llvm::Value *Scope = E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; + bool ShouldCastToIntPtrTy = true; switch (E->getOp()) { case AtomicExpr::AO__c11_atomic_init: @@ -1047,122 +1022,25 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { Dest = Atomics.castToAtomicIntPointer(Dest); } - // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . + bool PowerOf2Size = (Size & (Size - 1)) == 0; + bool UseLibcall = !PowerOf2Size || (Size > 16); + + // For atomics larger than 16 bytes, emit a libcall from the frontend. This + // avoids the overhead of dealing with excessively-large value types in IR. + // Non-power-of-2 values also lower to libcall here, as they are not currently + // permitted in IR instructions (although that constraint could be relaxed in + // the future). For other cases where a libcall is required on a given + // platform, we let the backend handle it (this includes handling for all of + // the size-optimized libcall variants, which are only valid up to 16 bytes.) + // + // See: https://llvm.org/docs/Atomics.html#libcalls-atomic if (UseLibcall) { - bool UseOptimizedLibcall = false; - switch (E->getOp()) { - case AtomicExpr::AO__c11_atomic_init: - case AtomicExpr::AO__opencl_atomic_init: - llvm_unreachable("Already handled above with EmitAtomicInit!"); - - case AtomicExpr::AO__atomic_fetch_add: - case AtomicExpr::AO__atomic_fetch_and: - case AtomicExpr::AO__atomic_fetch_max: - case AtomicExpr::AO__atomic_fetch_min: - case AtomicExpr::AO__atomic_fetch_nand: - case AtomicExpr::AO__atomic_fetch_or: - case AtomicExpr::AO__atomic_fetch_sub: - case AtomicExpr::AO__atomic_fetch_xor: - case AtomicExpr::AO__atomic_add_fetch: - case AtomicExpr::AO__atomic_and_fetch: - case AtomicExpr::AO__atomic_max_fetch: - case AtomicExpr::AO__atomic_min_fetch: - case AtomicExpr::AO__atomic_nand_fetch: - case AtomicExpr::AO__atomic_or_fetch: - case AtomicExpr::AO__atomic_sub_fetch: - case AtomicExpr::AO__atomic_xor_fetch: - case AtomicExpr::AO__c11_atomic_fetch_add: - case AtomicExpr::AO__c11_atomic_fetch_and: - case AtomicExpr::AO__c11_atomic_fetch_max: - case AtomicExpr::AO__c11_atomic_fetch_min: - case AtomicExpr::AO__c11_atomic_fetch_nand: - case AtomicExpr::AO__c11_atomic_fetch_or: - case AtomicExpr::AO__c11_atomic_fetch_sub: - case AtomicExpr::AO__c11_atomic_fetch_xor: - case AtomicExpr::AO__hip_atomic_fetch_add: - case AtomicExpr::AO__hip_atomic_fetch_and: - case AtomicExpr::AO__hip_atomic_fetch_max: - case AtomicExpr::AO__hip_atomic_fetch_min: - case AtomicExpr::AO__hip_atomic_fetch_or: - case AtomicExpr::AO__hip_atomic_fetch_sub: - case AtomicExpr::AO__hip_atomic_fetch_xor: - case AtomicExpr::AO__opencl_atomic_fetch_add: - case AtomicExpr::AO__opencl_atomic_fetch_and: - case AtomicExpr::AO__opencl_atomic_fetch_max: - case AtomicExpr::AO__opencl_atomic_fetch_min: - case AtomicExpr::AO__opencl_atomic_fetch_or: - case AtomicExpr::AO__opencl_atomic_fetch_sub: - case AtomicExpr::AO__opencl_atomic_fetch_xor: - case AtomicExpr::AO__scoped_atomic_fetch_add: - case AtomicExpr::AO__scoped_atomic_fetch_and: - case AtomicExpr::AO__scoped_atomic_fetch_max: - case AtomicExpr::AO__scoped_atomic_fetch_min: - case AtomicExpr::AO__scoped_atomic_fetch_nand: - case AtomicExpr::AO__scoped_atomic_fetch_or: - case AtomicExpr::AO__scoped_atomic_fetch_sub: - case AtomicExpr::AO__scoped_atomic_fetch_xor: - case AtomicExpr::AO__scoped_atomic_add_fetch: - case AtomicExpr::AO__scoped_atomic_and_fetch: - case AtomicExpr::AO__scoped_atomic_max_fetch: - case AtomicExpr::AO__scoped_atomic_min_fetch: - case AtomicExpr::AO__scoped_atomic_nand_fetch: - case AtomicExpr::AO__scoped_atomic_or_fetch: - case AtomicExpr::AO__scoped_atomic_sub_fetch: - case AtomicExpr::AO__scoped_atomic_xor_fetch: - // For these, only library calls for certain sizes exist. - UseOptimizedLibcall = true; - break; - - case AtomicExpr::AO__atomic_load: - case AtomicExpr::AO__atomic_store: - case AtomicExpr::AO__atomic_exchange: - case AtomicExpr::AO__atomic_compare_exchange: - case AtomicExpr::AO__scoped_atomic_load: - case AtomicExpr::AO__scoped_atomic_store: - case AtomicExpr::AO__scoped_atomic_exchange: - case AtomicExpr::AO__scoped_atomic_compare_exchange: - // Use the generic version if we don't know that the operand will be - // suitably aligned for the optimized version. - if (Misaligned) - break; - [[fallthrough]]; - case AtomicExpr::AO__atomic_load_n: - case AtomicExpr::AO__atomic_store_n: - case AtomicExpr::AO__atomic_exchange_n: - case AtomicExpr::AO__atomic_compare_exchange_n: - case AtomicExpr::AO__c11_atomic_load: - case AtomicExpr::AO__c11_atomic_store: - case AtomicExpr::AO__c11_atomic_exchange: - case AtomicExpr::AO__c11_atomic_compare_exchange_weak: - case AtomicExpr::AO__c11_atomic_compare_exchange_strong: - case AtomicExpr::AO__hip_atomic_load: - case AtomicExpr::AO__hip_atomic_store: - case AtomicExpr::AO__hip_atomic_exchange: - case AtomicExpr::AO__hip_atomic_compare_exchange_weak: - case AtomicExpr::AO__hip_atomic_compare_exchange_strong: - case AtomicExpr::AO__opencl_atomic_load: - case AtomicExpr::AO__opencl_atomic_store: - case AtomicExpr::AO__opencl_atomic_exchange: - case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: - case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: - case AtomicExpr::AO__scoped_atomic_load_n: - case AtomicExpr::AO__scoped_atomic_store_n: - case AtomicExpr::AO__scoped_atomic_exchange_n: - case AtomicExpr::AO__scoped_atomic_compare_exchange_n: - // Only use optimized library calls for sizes for which they exist. - // FIXME: Size == 16 optimized library functions exist too. - if (Size == 1 || Size == 2 || Size == 4 || Size == 8) - UseOptimizedLibcall = true; - break; - } - CallArgList Args; - if (!UseOptimizedLibcall) { - // For non-optimized library calls, the size is the first parameter - Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), - getContext().getSizeType()); - } - // Atomic address is the first or second parameter + // For non-optimized library calls, the size is the first parameter. + Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), + getContext().getSizeType()); + + // The atomic address is the second parameter. // The OpenCL atomic library functions only accept pointer arguments to // generic address space. auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { @@ -1178,17 +1056,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { *this, V, AS, LangAS::opencl_generic, DestType, false); }; - Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this), E->getPtr()->getType())), getContext().VoidPtrTy); + // The next 1-3 parameters are op-dependent. std::string LibCallName; - QualType LoweredMemTy = - MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; QualType RetTy; bool HaveRetTy = false; - llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; - bool PostOpMinMax = false; switch (E->getOp()) { case AtomicExpr::AO__c11_atomic_init: case AtomicExpr::AO__opencl_atomic_init: @@ -1199,8 +1074,6 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // and exchange. // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, // void *desired, int success, int failure) - // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, - // int success, int failure) case AtomicExpr::AO__atomic_compare_exchange: case AtomicExpr::AO__atomic_compare_exchange_n: case AtomicExpr::AO__c11_atomic_compare_exchange_weak: @@ -1214,17 +1087,17 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); + Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this), + E->getVal2()->getType())), + getContext().VoidPtrTy); Args.add(RValue::get(Order), getContext().IntTy); Order = OrderFail; break; // void __atomic_exchange(size_t size, void *mem, void *val, void *return, // int order) - // T __atomic_exchange_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: case AtomicExpr::AO__c11_atomic_exchange: @@ -1233,11 +1106,11 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__scoped_atomic_exchange: case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), + E->getVal1()->getType())), + getContext().VoidPtrTy); break; // void __atomic_store(size_t size, void *mem, void *val, int order) - // void __atomic_store_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: case AtomicExpr::AO__c11_atomic_store: @@ -1248,11 +1121,11 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), + E->getVal1()->getType())), + getContext().VoidPtrTy); break; // void __atomic_load(size_t size, void *mem, void *return, int order) - // T __atomic_load_N(T *mem, int order) case AtomicExpr::AO__atomic_load: case AtomicExpr::AO__atomic_load_n: case AtomicExpr::AO__c11_atomic_load: @@ -1262,183 +1135,86 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__scoped_atomic_load_n: LibCallName = "__atomic_load"; break; - // T __atomic_add_fetch_N(T *mem, T val, int order) - // T __atomic_fetch_add_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_add_fetch: case AtomicExpr::AO__scoped_atomic_add_fetch: - PostOp = llvm::Instruction::Add; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: case AtomicExpr::AO__scoped_atomic_fetch_add: - LibCallName = "__atomic_fetch_add"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - LoweredMemTy, E->getExprLoc(), TInfo.Width); - break; - // T __atomic_and_fetch_N(T *mem, T val, int order) - // T __atomic_fetch_and_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_and_fetch: case AtomicExpr::AO__scoped_atomic_and_fetch: - PostOp = llvm::Instruction::And; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_and: case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: case AtomicExpr::AO__scoped_atomic_fetch_and: - LibCallName = "__atomic_fetch_and"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); - break; - // T __atomic_or_fetch_N(T *mem, T val, int order) - // T __atomic_fetch_or_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_or_fetch: case AtomicExpr::AO__scoped_atomic_or_fetch: - PostOp = llvm::Instruction::Or; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_or: case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__scoped_atomic_fetch_or: - LibCallName = "__atomic_fetch_or"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); - break; - // T __atomic_sub_fetch_N(T *mem, T val, int order) - // T __atomic_fetch_sub_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_sub_fetch: case AtomicExpr::AO__scoped_atomic_sub_fetch: - PostOp = llvm::Instruction::Sub; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__scoped_atomic_fetch_sub: - LibCallName = "__atomic_fetch_sub"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - LoweredMemTy, E->getExprLoc(), TInfo.Width); - break; - // T __atomic_xor_fetch_N(T *mem, T val, int order) - // T __atomic_fetch_xor_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_xor_fetch: case AtomicExpr::AO__scoped_atomic_xor_fetch: - PostOp = llvm::Instruction::Xor; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_xor: case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__scoped_atomic_fetch_xor: - LibCallName = "__atomic_fetch_xor"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); - break; + case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__c11_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_nand_fetch: case AtomicExpr::AO__atomic_min_fetch: - case AtomicExpr::AO__scoped_atomic_min_fetch: - PostOpMinMax = true; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__c11_atomic_fetch_min: - case AtomicExpr::AO__scoped_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: - LibCallName = E->getValueType()->isSignedIntegerType() - ? "__atomic_fetch_min" - : "__atomic_fetch_umin"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - LoweredMemTy, E->getExprLoc(), TInfo.Width); - break; + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_min_fetch: case AtomicExpr::AO__atomic_max_fetch: - case AtomicExpr::AO__scoped_atomic_max_fetch: - PostOpMinMax = true; - [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: case AtomicExpr::AO__scoped_atomic_fetch_max: - LibCallName = E->getValueType()->isSignedIntegerType() - ? "__atomic_fetch_max" - : "__atomic_fetch_umax"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - LoweredMemTy, E->getExprLoc(), TInfo.Width); - break; - // T __atomic_nand_fetch_N(T *mem, T val, int order) - // T __atomic_fetch_nand_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_nand_fetch: - case AtomicExpr::AO__scoped_atomic_nand_fetch: - PostOp = llvm::Instruction::And; // the NOT is special cased below - [[fallthrough]]; - case AtomicExpr::AO__atomic_fetch_nand: - case AtomicExpr::AO__c11_atomic_fetch_nand: - case AtomicExpr::AO__scoped_atomic_fetch_nand: - LibCallName = "__atomic_fetch_nand"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), - MemTy, E->getExprLoc(), TInfo.Width); - break; + case AtomicExpr::AO__scoped_atomic_max_fetch: + llvm_unreachable("Integral atomic operations always become atomicrmw!"); } if (E->isOpenCL()) { - LibCallName = std::string("__opencl") + - StringRef(LibCallName).drop_front(1).str(); - + LibCallName = + std::string("__opencl") + StringRef(LibCallName).drop_front(1).str(); } - // Optimized functions have the size in their name. - if (UseOptimizedLibcall) - LibCallName += "_" + llvm::utostr(Size); // By default, assume we return a value of the atomic type. if (!HaveRetTy) { - if (UseOptimizedLibcall) { - // Value is returned directly. - // The function returns an appropriately sized integer type. - RetTy = getContext().getIntTypeForBitwidth( - getContext().toBits(TInfo.Width), /*Signed=*/false); - } else { - // Value is returned through parameter before the order. - RetTy = getContext().VoidTy; - Args.add(RValue::get(Dest.getPointer()), getContext().VoidPtrTy); - } + // Value is returned through parameter before the order. + RetTy = getContext().VoidTy; + Args.add(RValue::get( + CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)), + getContext().VoidPtrTy); } - // order is always the last parameter + // Order is always the last parameter. Args.add(RValue::get(Order), getContext().IntTy); if (E->isOpenCL()) Args.add(RValue::get(Scope), getContext().IntTy); - // PostOp is only needed for the atomic_*_fetch operations, and - // thus is only needed for and implemented in the - // UseOptimizedLibcall codepath. - assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); - RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); // The value is returned directly from the libcall. if (E->isCmpXChg()) return Res; - // The value is returned directly for optimized libcalls but the expr - // provided an out-param. - if (UseOptimizedLibcall && Res.getScalarVal()) { - llvm::Value *ResVal = Res.getScalarVal(); - if (PostOpMinMax) { - llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); - ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), - E->getValueType()->isSignedIntegerType(), - ResVal, LoadVal1); - } else if (PostOp) { - llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); - ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); - } - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || - E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) - ResVal = Builder.CreateNot(ResVal); - - Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType())); - } - if (RValTy->isVoidType()) return RValue::get(nullptr); @@ -1625,12 +1401,26 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr, LVal.getBaseInfo(), TBAAAccessInfo())); } -RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, - AggValueSlot ResultSlot, - SourceLocation Loc, - bool AsValue) const { +/// Return true if \param ValTy is a type that should be casted to integer +/// around the atomic memory operation. If \param CmpXchg is true, then the +/// cast of a floating point type is made as that instruction can not have +/// floating point operands. TODO: Allow compare-and-exchange and FP - see +/// comment in AtomicExpandPass.cpp. +static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { + if (ValTy->isFloatingPointTy()) + return ValTy->isX86_FP80Ty() || CmpXchg; + return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); +} + +RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val, + AggValueSlot ResultSlot, + SourceLocation Loc, bool AsValue, + bool CmpXchg) const { // Try not to in some easy cases. - assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); + assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() || + Val->getType()->isIEEELikeFPTy()) && + "Expected integer, pointer or floating point value when converting " + "result."); if (getEvaluationKind() == TEK_Scalar && (((!LVal.isBitField() || LVal.getBitFieldInfo().Size == ValueSizeInBits) && @@ -1639,13 +1429,13 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, auto *ValTy = AsValue ? CGF.ConvertTypeForMem(ValueTy) : getAtomicAddress().getElementType(); - if (ValTy->isIntegerTy()) { - assert(IntVal->getType() == ValTy && "Different integer types."); - return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); - } else if (ValTy->isPointerTy()) - return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); - else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) - return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); + if (!shouldCastToInt(ValTy, CmpXchg)) { + assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) && + "Different integer types."); + return RValue::get(CGF.EmitFromMemory(Val, ValueTy)); + } + if (llvm::CastInst::isBitCastable(Val->getType(), ValTy)) + return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy)); } // Create a temporary. This needs to be big enough to hold the @@ -1662,8 +1452,7 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, // Slam the integer into the temporary. Address CastTemp = castToAtomicIntPointer(Temp); - CGF.Builder.CreateStore(IntVal, CastTemp) - ->setVolatile(TempIsVolatile); + CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile); return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); } @@ -1682,9 +1471,11 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, } llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, - bool IsVolatile) { + bool IsVolatile, bool CmpXchg) { // Okay, we're doing this natively. - Address Addr = getAtomicAddressAsAtomicIntPointer(); + Address Addr = getAtomicAddress(); + if (shouldCastToInt(Addr.getElementType(), CmpXchg)) + Addr = castToAtomicIntPointer(Addr); llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); Load->setAtomic(AO); @@ -1736,7 +1527,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, } else TempAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile); // Okay, turn that back into the original value or whole atomic (for // non-simple lvalues) type. @@ -1752,7 +1543,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, // Okay, turn that back into the original value or atomic (for non-simple // lvalues) type. - return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); + return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue); } /// Emit a load from an l-value of atomic type. Note that the r-value @@ -1812,23 +1603,26 @@ Address AtomicInfo::materializeRValue(RValue rvalue) const { LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); AtomicInfo Atomics(CGF, TempLV); Atomics.emitCopyIntoMemory(rvalue); - return TempLV.getAddress(CGF); + return TempLV.getAddress(); } -llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { +llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const { + if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) + return RVal.getScalarVal(); + return nullptr; +} + +llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const { // If we've got a scalar value of the right size, try to avoid going - // through memory. - if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { - llvm::Value *Value = RVal.getScalarVal(); - if (isa<llvm::IntegerType>(Value->getType())) + // through memory. Floats get casted if needed by AtomicExpandPass. + if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) { + if (!shouldCastToInt(Value->getType(), CmpXchg)) return CGF.EmitToMemory(Value, ValueTy); else { llvm::IntegerType *InputIntTy = llvm::IntegerType::get( CGF.getLLVMContext(), LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); - if (isa<llvm::PointerType>(Value->getType())) - return CGF.Builder.CreatePtrToInt(Value, InputIntTy); - else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) + if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) return CGF.Builder.CreateBitCast(Value, InputIntTy); } } @@ -1889,9 +1683,9 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( if (shouldUseLibcall()) { // Produce a source address. Address ExpectedAddr = materializeRValue(Expected); - Address DesiredAddr = materializeRValue(Desired); - auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, Success, Failure); return std::make_pair( convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), @@ -1901,13 +1695,14 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( // If we've got a scalar value of the right size, try to avoid going // through memory. - auto *ExpectedVal = convertRValueToInt(Expected); - auto *DesiredVal = convertRValueToInt(Desired); + auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true); + auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true); auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, Failure, IsWeak); return std::make_pair( - ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), - SourceLocation(), /*AsValue=*/false), + ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false, + /*CmpXchg=*/true), Res.second); } @@ -1973,7 +1768,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall( Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1987,10 +1782,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall( AggValueSlot::ignored(), SourceLocation(), /*AsValue=*/false); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), - AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -2001,7 +1796,7 @@ void AtomicInfo::EmitAtomicUpdateOp( auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); // Do the atomic load. - auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); + auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true); // For non-simple lvalues perform compare-and-swap procedure. auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); @@ -2011,13 +1806,18 @@ void AtomicInfo::EmitAtomicUpdateOp( /*NumReservedValues=*/2); PHI->addIncoming(OldVal, CurBB); Address NewAtomicAddr = CreateTempAlloca(); - Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); + Address NewAtomicIntAddr = + shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true) + ? castToAtomicIntPointer(NewAtomicAddr) + : NewAtomicAddr; + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || requiresMemSetZero(getAtomicAddress().getElementType())) { CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); } - auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), - SourceLocation(), /*AsValue=*/false); + auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false, + /*CmpXchg=*/true); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); // Try to write new value using cmpxchg operation. @@ -2059,7 +1859,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -2070,10 +1870,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, CGF.Builder.CreateStore(OldVal, DesiredAddr); } EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), - AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -2083,7 +1883,7 @@ void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); // Do the atomic load. - auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); + auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true); // For non-simple lvalues perform compare-and-swap procedure. auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); @@ -2151,7 +1951,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, // maybe for address-space qualification. assert(!rvalue.isAggregate() || rvalue.getAggregateAddress().getElementType() == - dest.getAddress(*this).getElementType()); + dest.getAddress().getElementType()); AtomicInfo atomics(*this, dest); LValue LVal = atomics.getAtomicLValue(); @@ -2173,7 +1973,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, args.add(RValue::get(atomics.getAtomicSizeValue()), getContext().getSizeType()); args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); - args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy); + args.add(RValue::get(srcAddr.emitRawPointer(*this)), + getContext().VoidPtrTy); args.add( RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), getContext().IntTy); @@ -2182,13 +1983,17 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, } // Okay, we're doing this natively. - llvm::Value *intValue = atomics.convertRValueToInt(rvalue); + llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue); // Do the atomic store. - Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress()); - intValue = Builder.CreateIntCast( - intValue, addr.getElementType(), /*isSigned=*/false); - llvm::StoreInst *store = Builder.CreateStore(intValue, addr); + Address Addr = atomics.getAtomicAddress(); + if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue)) + if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) { + Addr = atomics.castToAtomicIntPointer(Addr); + ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(), + /*isSigned=*/false); + } + llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr); if (AO == llvm::AtomicOrdering::Acquire) AO = llvm::AtomicOrdering::Monotonic; @@ -2219,10 +2024,10 @@ std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( // maybe for address-space qualification. assert(!Expected.isAggregate() || Expected.getAggregateAddress().getElementType() == - Obj.getAddress(*this).getElementType()); + Obj.getAddress().getElementType()); assert(!Desired.isAggregate() || Desired.getAggregateAddress().getElementType() == - Obj.getAddress(*this).getElementType()); + Obj.getAddress().getElementType()); AtomicInfo Atomics(*this, Obj); return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, @@ -2263,7 +2068,7 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { // Evaluate the expression directly into the destination. AggValueSlot slot = AggValueSlot::forLValue( - dest, *this, AggValueSlot::IsNotDestructed, + dest, AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap, Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.cpp index 0cbace7b7f7b..066139b1c78c 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.cpp @@ -36,7 +36,8 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false), NoEscape(false), HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false), CapturesNonExternalType(false), - LocalAddress(Address::invalid()), StructureType(nullptr), Block(block) { + LocalAddress(RawAddress::invalid()), StructureType(nullptr), + Block(block) { // Skip asm prefix, if any. 'name' is usually taken directly from // the mangled name of the enclosing function. @@ -120,11 +121,15 @@ static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo, Name += "_"; } - std::string TypeAtEncoding = - CGM.getContext().getObjCEncodingForBlock(BlockInfo.getBlockExpr()); - /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms as - /// a separator between symbol name and symbol version. - std::replace(TypeAtEncoding.begin(), TypeAtEncoding.end(), '@', '\1'); + std::string TypeAtEncoding; + + if (!CGM.getCodeGenOpts().DisableBlockSignatureString) { + TypeAtEncoding = + CGM.getContext().getObjCEncodingForBlock(BlockInfo.getBlockExpr()); + /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms + /// as a separator between symbol name and symbol version. + std::replace(TypeAtEncoding.begin(), TypeAtEncoding.end(), '@', '\1'); + } Name += "e" + llvm::to_string(TypeAtEncoding.size()) + "_" + TypeAtEncoding; Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, BlockInfo); return Name; @@ -200,9 +205,13 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, } // Signature. Mandatory ObjC-style method descriptor @encode sequence. - std::string typeAtEncoding = - CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr()); - elements.add(CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer()); + if (CGM.getCodeGenOpts().DisableBlockSignatureString) { + elements.addNullPointer(i8p); + } else { + std::string typeAtEncoding = + CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr()); + elements.add(CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer()); + } // GC layout. if (C.getLangOpts().ObjC) { @@ -576,7 +585,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, // First, 'this'. if (block->capturesCXXThis()) { - assert(CGF && CGF->CurFuncDecl && isa<CXXMethodDecl>(CGF->CurFuncDecl) && + assert(CGF && isa_and_nonnull<CXXMethodDecl>(CGF->CurFuncDecl) && "Can't capture 'this' outside a method"); QualType thisType = cast<CXXMethodDecl>(CGF->CurFuncDecl)->getThisType(); @@ -794,7 +803,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Otherwise, we have to emit this as a local block. - Address blockAddr = blockInfo.LocalAddress; + RawAddress blockAddr = blockInfo.LocalAddress; assert(blockAddr.isValid() && "block has no address!"); llvm::Constant *isa; @@ -813,7 +822,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { descriptor = buildBlockDescriptor(CGM, blockInfo); // Compute the initial on-stack block flags. - flags = BLOCK_HAS_SIGNATURE; + if (!CGM.getCodeGenOpts().DisableBlockSignatureString) + flags = BLOCK_HAS_SIGNATURE; if (blockInfo.HasCapturedVariableLayout) flags |= BLOCK_HAS_EXTENDED_LAYOUT; if (blockInfo.NeedsCopyDispose) @@ -926,7 +936,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { /*RefersToEnclosingVariableOrCapture*/ CI.isNested(), type.getNonReferenceType(), VK_LValue, SourceLocation()); - src = EmitDeclRefLValue(&declRef).getAddress(*this); + src = EmitDeclRefLValue(&declRef).getAddress(); }; // For byrefs, we just write the pointer to the byref struct into @@ -939,7 +949,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (CI.isNested()) byrefPointer = Builder.CreateLoad(src, "byref.capture"); else - byrefPointer = src.getPointer(); + byrefPointer = src.emitRawPointer(*this); // Write that void* into the capture field. Builder.CreateStore(byrefPointer, blockField); @@ -961,10 +971,10 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { } // If it's a reference variable, copy the reference into the block field. - } else if (type->isReferenceType()) { - Builder.CreateStore(src.getPointer(), blockField); + } else if (type->getAs<ReferenceType>()) { + Builder.CreateStore(src.emitRawPointer(*this), blockField); - // If type is const-qualified, copy the value into the block field. + // If type is const-qualified, copy the value into the block field. } else if (type.isConstQualified() && type.getObjCLifetime() == Qualifiers::OCL_Strong && CGM.getCodeGenOpts().OptimizationLevel != 0) { @@ -1299,7 +1309,9 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, fields.add(CGM.getNSConcreteGlobalBlock()); // __flags - BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE; + BlockFlags flags = BLOCK_IS_GLOBAL; + if (!CGM.getCodeGenOpts().DisableBlockSignatureString) + flags |= BLOCK_HAS_SIGNATURE; if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET; @@ -1377,7 +1389,7 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, // Allocate a stack slot like for any local variable to guarantee optimal // debug info at -O0. The mem2reg pass will eliminate it when optimizing. - Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); + RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); Builder.CreateStore(arg, alloc); if (CGDebugInfo *DI = getDebugInfo()) { if (CGM.getCodeGenOpts().hasReducedDebugInfo()) { @@ -1446,7 +1458,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( selfTy = getContext().getPointerType(getContext().getAddrSpaceQualType( getContext().VoidTy, LangAS::opencl_generic)); - IdentifierInfo *II = &CGM.getContext().Idents.get(".block_descriptor"); + const IdentifierInfo *II = &CGM.getContext().Idents.get(".block_descriptor"); ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl), SourceLocation(), II, selfTy, @@ -1497,7 +1509,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( // frame setup instruction by llvm::DwarfDebug::beginFunction(). auto NL = ApplyDebugLocation::CreateEmpty(*this); Builder.CreateStore(BlockPointer, Alloca); - BlockPointerDbgLoc = Alloca.getPointer(); + BlockPointerDbgLoc = Alloca.emitRawPointer(*this); } // If we have a C++ 'this' reference, go ahead and force it into @@ -1540,7 +1552,10 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( llvm::BasicBlock *resume = Builder.GetInsertBlock(); // Go back to the entry. - ++entry_ptr; + if (entry_ptr->getNextNonDebugInstruction()) + entry_ptr = entry_ptr->getNextNonDebugInstruction()->getIterator(); + else + entry_ptr = entry->end(); Builder.SetInsertPoint(entry, entry_ptr); // Emit debug information for all the DeclRefExprs. @@ -1554,8 +1569,8 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); if (capture.isConstant()) { auto addr = LocalDeclMap.find(variable)->second; - (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(), - Builder); + (void)DI->EmitDeclareOfAutoVariable( + variable, addr.emitRawPointer(*this), Builder); continue; } @@ -1659,7 +1674,7 @@ struct CallBlockRelease final : EHScopeStack::Cleanup { if (LoadBlockVarAddr) { BlockVarAddr = CGF.Builder.CreateLoad(Addr); } else { - BlockVarAddr = Addr.getPointer(); + BlockVarAddr = Addr.emitRawPointer(CGF); } CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow); @@ -1959,13 +1974,15 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { // it. It's not quite worth the annoyance to avoid creating it in the // first place. if (!needsEHCleanup(captureType.isDestructedType())) - cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent(); + if (auto *I = + cast_or_null<llvm::Instruction>(dstField.getBasePointer())) + I->eraseFromParent(); } break; } case BlockCaptureEntityKind::BlockObject: { llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src"); - llvm::Value *dstAddr = dstField.getPointer(); + llvm::Value *dstAddr = dstField.emitRawPointer(*this); llvm::Value *args[] = { dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) }; @@ -2136,7 +2153,7 @@ public: llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags); llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign(); - llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal }; + llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal}; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2693,7 +2710,8 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { storeHeaderField(V, getPointerSize(), "byref.isa"); // Store the address of the variable into its own forwarding pointer. - storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding"); + storeHeaderField(addr.emitRawPointer(*this), getPointerSize(), + "byref.forwarding"); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are @@ -2784,7 +2802,7 @@ static void configureBlocksRuntimeObject(CodeGenModule &CGM, auto *GV = cast<llvm::GlobalValue>(C->stripPointerCasts()); if (CGM.getTarget().getTriple().isOSBinFormatCOFF()) { - IdentifierInfo &II = CGM.getContext().Idents.get(C->getName()); + const IdentifierInfo &II = CGM.getContext().Idents.get(C->getName()); TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl(); DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.h b/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.h index 4ef1ae9f3365..8d10c4f69b20 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGBlocks.h @@ -271,7 +271,8 @@ public: /// The block's captures. Non-constant captures are sorted by their offsets. llvm::SmallVector<Capture, 4> SortedCaptures; - Address LocalAddress; + // Currently we assume that block-pointer types are never signed. + RawAddress LocalAddress; llvm::StructType *StructureType; const BlockDecl *Block; const BlockExpr *BlockExpression; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGBuilder.h b/contrib/llvm-project/clang/lib/CodeGen/CGBuilder.h index bf5ab171d720..5d59d5a4ae2c 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGBuilder.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGBuilder.h @@ -10,7 +10,9 @@ #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H #include "Address.h" +#include "CGValue.h" #include "CodeGenTypeCache.h" +#include "llvm/Analysis/Utils/Local.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Type.h" @@ -18,19 +20,21 @@ namespace clang { namespace CodeGen { +class CGBuilderTy; class CodeGenFunction; /// This is an IRBuilder insertion helper that forwards to /// CodeGenFunction::InsertHelper, which adds necessary metadata to /// instructions. class CGBuilderInserter final : public llvm::IRBuilderDefaultInserter { + friend CGBuilderTy; + public: CGBuilderInserter() = default; explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {} /// This forwards to CodeGenFunction::InsertHelper. void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name, - llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const override; private: @@ -43,10 +47,42 @@ typedef llvm::IRBuilder<llvm::ConstantFolder, CGBuilderInserterTy> CGBuilderBaseTy; class CGBuilderTy : public CGBuilderBaseTy { + friend class Address; + /// Storing a reference to the type cache here makes it a lot easier /// to build natural-feeling, target-specific IR. const CodeGenTypeCache &TypeCache; + CodeGenFunction *getCGF() const { return getInserter().CGF; } + + llvm::Value *emitRawPointerFromAddress(Address Addr) const { + return Addr.getBasePointer(); + } + + template <bool IsInBounds> + Address createConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, + const llvm::Twine &Name) { + const llvm::DataLayout &DL = BB->getDataLayout(); + llvm::GetElementPtrInst *GEP; + if (IsInBounds) + GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32( + Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, + Name)); + else + GEP = cast<llvm::GetElementPtrInst>(CreateConstGEP2_32( + Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, + Name)); + llvm::APInt Offset( + DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, + /*isSigned=*/true); + if (!GEP->accumulateConstantOffset(DL, Offset)) + llvm_unreachable("offset of GEP with constants is always computable"); + return Address(GEP, GEP->getResultElementType(), + Addr.getAlignment().alignmentAtOffset( + CharUnits::fromQuantity(Offset.getSExtValue())), + IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull); + } + public: CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C) : CGBuilderBaseTy(C), TypeCache(TypeCache) {} @@ -69,20 +105,22 @@ public: // Note that we intentionally hide the CreateLoad APIs that don't // take an alignment. llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + return CreateAlignedLoad(Addr.getElementType(), + emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, const char *Name) { // This overload is required to prevent string literals from // ending up in the IsVolatile overload. - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + return CreateAlignedLoad(Addr.getElementType(), + emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), - Addr.getAlignment().getAsAlign(), IsVolatile, - Name); + return CreateAlignedLoad( + Addr.getElementType(), emitRawPointerFromAddress(Addr), + Addr.getAlignment().getAsAlign(), IsVolatile, Name); } using CGBuilderBaseTy::CreateAlignedLoad; @@ -96,7 +134,7 @@ public: // take an alignment. llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile = false) { - return CreateAlignedStore(Val, Addr.getPointer(), + return CreateAlignedStore(Val, emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), IsVolatile); } @@ -132,33 +170,41 @@ public: llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { return CGBuilderBaseTy::CreateAtomicCmpXchg( - Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(), - SuccessOrdering, FailureOrdering, SSID); + Addr.emitRawPointer(*getCGF()), Cmp, New, + Addr.getAlignment().getAsAlign(), SuccessOrdering, FailureOrdering, + SSID); } llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { - return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val, - Addr.getAlignment().getAsAlign(), - Ordering, SSID); + return CGBuilderBaseTy::CreateAtomicRMW( + Op, Addr.emitRawPointer(*getCGF()), Val, + Addr.getAlignment().getAsAlign(), Ordering, SSID); } using CGBuilderBaseTy::CreateAddrSpaceCast; Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, + llvm::Type *ElementTy, const llvm::Twine &Name = "") { - return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name), - Addr.isKnownNonNull()); + if (!Addr.hasOffset()) + return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name), + ElementTy, Addr.getAlignment(), Addr.getPointerAuthInfo(), + /*Offset=*/nullptr, Addr.isKnownNonNull()); + // Eagerly force a raw address if these is an offset. + return RawAddress( + CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name), + ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name = "") { - llvm::Value *Ptr = - CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name); - return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); + if (Addr.getType()->getAddressSpace() == Ty->getPointerAddressSpace()) + return Addr.withElementType(ElementTy); + return CreateAddrSpaceCast(Addr, Ty, ElementTy, Name); } /// Given @@ -172,14 +218,15 @@ public: Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name = "") { llvm::StructType *ElTy = cast<llvm::StructType>(Addr.getElementType()); - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + const llvm::DataLayout &DL = BB->getDataLayout(); const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address( - CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull()); + return Address(CreateStructGEP(Addr.getElementType(), Addr.getBasePointer(), + Index, Name), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } /// Given @@ -193,12 +240,12 @@ public: Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name = "") { llvm::ArrayType *ElTy = cast<llvm::ArrayType>(Addr.getElementType()); - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + const llvm::DataLayout &DL = BB->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType())); return Address( - CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), + CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), {getSize(CharUnits::Zero()), getSize(Index)}, Name), ElTy->getElementType(), Addr.getAlignment().alignmentAtOffset(Index * EltSize), @@ -213,13 +260,13 @@ public: Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name = "") { llvm::Type *ElTy = Addr.getElementType(); - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + const llvm::DataLayout &DL = BB->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Index), Name), - ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), - Addr.isKnownNonNull()); + return Address( + CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), + ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), + Addr.isKnownNonNull()); } /// Given @@ -229,110 +276,133 @@ public: /// where i64 is actually the target word size. Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name = "") { - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); - CharUnits EltSize = - CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); + llvm::Type *ElTy = Addr.getElementType(); + const llvm::DataLayout &DL = BB->getDataLayout(); + CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Index), Name), + return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Index * EltSize), - NotKnownNonNull); + Addr.getAlignment().alignmentAtOffset(Index * EltSize)); } /// Create GEP with single dynamic index. The address alignment is reduced /// according to the element size. using CGBuilderBaseTy::CreateGEP; - Address CreateGEP(Address Addr, llvm::Value *Index, + Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name = "") { - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + const llvm::DataLayout &DL = BB->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); return Address( - CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), + CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name), Addr.getElementType(), - Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull); + Addr.getAlignment().alignmentOfArrayElement(EltSize)); } /// Given a pointer to i8, adjust it by a given constant offset. Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Offset), Name), - Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset), - Addr.isKnownNonNull()); + return Address( + CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), + getSize(Offset), Name), + Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } + Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), + return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(), getSize(Offset), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset), - NotKnownNonNull); + Addr.getAlignment().alignmentAtOffset(Offset)); } using CGBuilderBaseTy::CreateConstInBoundsGEP2_32; Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name = "") { - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + return createConstGEP2_32<true>(Addr, Idx0, Idx1, Name); + } - auto *GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32( - Addr.getElementType(), Addr.getPointer(), Idx0, Idx1, Name)); - llvm::APInt Offset( - DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, - /*isSigned=*/true); - if (!GEP->accumulateConstantOffset(DL, Offset)) - llvm_unreachable("offset of GEP with constants is always computable"); - return Address(GEP, GEP->getResultElementType(), - Addr.getAlignment().alignmentAtOffset( - CharUnits::fromQuantity(Offset.getSExtValue())), - Addr.isKnownNonNull()); + using CGBuilderBaseTy::CreateConstGEP2_32; + Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, + const llvm::Twine &Name = "") { + return createConstGEP2_32<false>(Addr, Idx0, Idx1, Name); + } + + Address CreateGEP(Address Addr, ArrayRef<llvm::Value *> IdxList, + llvm::Type *ElementType, CharUnits Align, + const Twine &Name = "") { + llvm::Value *Ptr = emitRawPointerFromAddress(Addr); + return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name), + ElementType, Align); + } + + using CGBuilderBaseTy::CreateInBoundsGEP; + Address CreateInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList, + llvm::Type *ElementType, CharUnits Align, + const Twine &Name = "") { + return RawAddress(CreateInBoundsGEP(Addr.getElementType(), + emitRawPointerFromAddress(Addr), + IdxList, Name), + ElementType, Align, Addr.isKnownNonNull()); + } + + using CGBuilderBaseTy::CreateIsNull; + llvm::Value *CreateIsNull(Address Addr, const Twine &Name = "") { + if (!Addr.hasOffset()) + return CreateIsNull(Addr.getBasePointer(), Name); + // The pointer isn't null if Addr has an offset since offsets can always + // be applied inbound. + return llvm::ConstantInt::getFalse(Context); } using CGBuilderBaseTy::CreateMemCpy; llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), Size, - IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size, bool IsVolatile = false) { - return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), Size, - IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } using CGBuilderBaseTy::CreateMemCpyInline; llvm::CallInst *CreateMemCpyInline(Address Dest, Address Src, uint64_t Size) { - return CreateMemCpyInline( - Dest.getPointer(), Dest.getAlignment().getAsAlign(), Src.getPointer(), - Src.getAlignment().getAsAlign(), getInt64(Size)); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpyInline(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), getInt64(Size)); } using CGBuilderBaseTy::CreateMemMove; llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), - Size, IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemMove(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } using CGBuilderBaseTy::CreateMemSet; llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemSet(Dest.getPointer(), Value, Size, + return CreateMemSet(emitRawPointerFromAddress(Dest), Value, Size, Dest.getAlignment().getAsAlign(), IsVolatile); } using CGBuilderBaseTy::CreateMemSetInline; llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size) { - return CreateMemSetInline(Dest.getPointer(), + return CreateMemSetInline(emitRawPointerFromAddress(Dest), Dest.getAlignment().getAsAlign(), Value, getInt64(Size)); } @@ -342,20 +412,35 @@ public: unsigned FieldIndex, llvm::MDNode *DbgInfo) { llvm::StructType *ElTy = cast<llvm::StructType>(Addr.getElementType()); - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + const llvm::DataLayout &DL = BB->getDataLayout(); const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(), - Index, FieldIndex, DbgInfo), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset)); + return Address( + CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr), + Index, FieldIndex, DbgInfo), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset)); + } + + using CGBuilderBaseTy::CreatePreserveUnionAccessIndex; + Address CreatePreserveUnionAccessIndex(Address Addr, unsigned FieldIndex, + llvm::MDNode *DbgInfo) { + Addr.replaceBasePointer(CreatePreserveUnionAccessIndex( + Addr.getBasePointer(), FieldIndex, DbgInfo)); + return Addr; } using CGBuilderBaseTy::CreateLaunderInvariantGroup; Address CreateLaunderInvariantGroup(Address Addr) { - return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()), - Addr.isKnownNonNull()); + Addr.replaceBasePointer(CreateLaunderInvariantGroup(Addr.getBasePointer())); + return Addr; + } + + using CGBuilderBaseTy::CreateStripInvariantGroup; + Address CreateStripInvariantGroup(Address Addr) { + Addr.replaceBasePointer(CreateStripInvariantGroup(Addr.getBasePointer())); + return Addr; } }; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp index a4f26a6f0eb1..86d47054615e 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp @@ -13,6 +13,7 @@ #include "ABIInfo.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" +#include "CGHLSLRuntime.h" #include "CGObjCRuntime.h" #include "CGOpenCLRuntime.h" #include "CGRecordLayout.h" @@ -44,6 +45,7 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsARM.h" #include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/IR/IntrinsicsDirectX.h" #include "llvm/IR/IntrinsicsHexagon.h" #include "llvm/IR/IntrinsicsNVPTX.h" #include "llvm/IR/IntrinsicsPowerPC.h" @@ -55,6 +57,7 @@ #include "llvm/IR/IntrinsicsX86.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/MatrixBuilder.h" +#include "llvm/IR/MemoryModelRelaxationAnnotations.h" #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/ScopedPrinter.h" @@ -511,8 +514,8 @@ static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); if (CGF.Builder.getIsFPConstrained()) { - CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType()); return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 }); } else { @@ -528,8 +531,8 @@ static Value *emitBinaryExpMaybeConstrainedFPBuiltin( llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); if (CGF.Builder.getIsFPConstrained()) { - CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, {Src0->getType(), Src1->getType()}); return CGF.Builder.CreateConstrainedFPCall(F, {Src0, Src1}); @@ -549,8 +552,8 @@ static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); if (CGF.Builder.getIsFPConstrained()) { - CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType()); return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 }); } else { @@ -578,38 +581,19 @@ static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, return CGF.Builder.CreateCall(F, Args); } -// Emit a simple mangled intrinsic that has 1 argument and a return type -// matching the argument type. -static Value *emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, - unsigned IntrinsicID, - llvm::StringRef Name = "") { - llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); - - Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall(F, Src0, Name); -} - -// Emit an intrinsic that has 2 operands of the same type as its result. -static Value *emitBinaryBuiltin(CodeGenFunction &CGF, - const CallExpr *E, - unsigned IntrinsicID) { - llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); - llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); - - Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall(F, { Src0, Src1 }); -} - -// Emit an intrinsic that has 3 operands of the same type as its result. -static Value *emitTernaryBuiltin(CodeGenFunction &CGF, - const CallExpr *E, - unsigned IntrinsicID) { - llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); - llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); - llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); - - Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 }); +// Emit a simple intrinsic that has N scalar arguments and a return type +// matching the argument type. It is assumed that only the first argument is +// overloaded. +template <unsigned N> +Value *emitBuiltinWithOneOverloadedType(CodeGenFunction &CGF, const CallExpr *E, + unsigned IntrinsicID, + llvm::StringRef Name = "") { + static_assert(N, "expect non-empty argument"); + SmallVector<Value *, N> Args; + for (unsigned I = 0; I < N; ++I) + Args.push_back(CGF.EmitScalarExpr(E->getArg(I))); + Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Args[0]->getType()); + return CGF.Builder.CreateCall(F, Args, Name); } // Emit an intrinsic that has 1 float or double operand, and 1 integer. @@ -702,8 +686,36 @@ static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue) { + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E); CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD)); - return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot()); + RValue Call = + CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot()); + + // Check the supported intrinsic. + if (unsigned BuiltinID = FD->getBuiltinID()) { + auto IsErrnoIntrinsic = [&]() -> unsigned { + switch (BuiltinID) { + case Builtin::BIexpf: + case Builtin::BI__builtin_expf: + case Builtin::BI__builtin_expf128: + return true; + } + // TODO: support more FP math libcalls + return false; + }(); + + // Restrict to target with errno, for example, MacOS doesn't set errno. + if (IsErrnoIntrinsic && CGF.CGM.getLangOpts().MathErrno && + !CGF.Builder.getIsFPConstrained()) { + ASTContext &Context = CGF.getContext(); + // Emit "int" TBAA metadata on FP math libcalls. + clang::QualType IntTy = Context.IntTy; + TBAAAccessInfo TBAAInfo = CGF.CGM.getTBAAAccessInfo(IntTy); + Instruction *Inst = cast<llvm::Instruction>(Call.getScalarVal()); + CGF.CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo); + } + } + return Call; } /// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* @@ -730,17 +742,14 @@ static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, return CGF.Builder.CreateExtractValue(Tmp, 0); } -static Value *emitRangedBuiltin(CodeGenFunction &CGF, - unsigned IntrinsicID, +static Value *emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high) { - llvm::MDBuilder MDHelper(CGF.getLLVMContext()); - llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high)); - Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {}); - llvm::Instruction *Call = CGF.Builder.CreateCall(F); - Call->setMetadata(llvm::LLVMContext::MD_range, RNode); - Call->setMetadata(llvm::LLVMContext::MD_noundef, - llvm::MDNode::get(CGF.getLLVMContext(), std::nullopt)); - return Call; + Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {}); + llvm::CallInst *Call = CGF.Builder.CreateCall(F); + llvm::ConstantRange CR(APInt(32, low), APInt(32, high)); + Call->addRangeRetAttr(CR); + Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef); + return Call; } namespace { @@ -791,7 +800,8 @@ EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) { Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) { Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend; - return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue); + return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}), + ArgValue); } /// Checks if using the result of __builtin_object_size(p, @p From) in place of @@ -819,33 +829,37 @@ CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); } -const FieldDecl *CodeGenFunction::FindFlexibleArrayMemberField( - ASTContext &Ctx, const RecordDecl *RD, StringRef Name, uint64_t &Offset) { +const FieldDecl *CodeGenFunction::FindFlexibleArrayMemberFieldAndOffset( + ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, + uint64_t &Offset) { const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = getLangOpts().getStrictFlexArraysLevel(); - unsigned FieldNo = 0; - bool IsUnion = RD->isUnion(); + uint32_t FieldNo = 0; + + if (RD->isImplicit()) + return nullptr; - for (const Decl *D : RD->decls()) { - if (const auto *Field = dyn_cast<FieldDecl>(D); - Field && (Name.empty() || Field->getNameAsString() == Name) && + for (const FieldDecl *FD : RD->fields()) { + if ((!FAMDecl || FD == FAMDecl) && Decl::isFlexibleArrayMemberLike( - Ctx, Field, Field->getType(), StrictFlexArraysLevel, + Ctx, FD, FD->getType(), StrictFlexArraysLevel, /*IgnoreTemplateOrMacroSubstitution=*/true)) { const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD); Offset += Layout.getFieldOffset(FieldNo); - return Field; + return FD; } - if (const auto *Record = dyn_cast<RecordDecl>(D)) - if (const FieldDecl *Field = - FindFlexibleArrayMemberField(Ctx, Record, Name, Offset)) { + QualType Ty = FD->getType(); + if (Ty->isRecordType()) { + if (const FieldDecl *Field = FindFlexibleArrayMemberFieldAndOffset( + Ctx, Ty->getAsRecordDecl(), FAMDecl, Offset)) { const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD); Offset += Layout.getFieldOffset(FieldNo); return Field; } + } - if (!IsUnion && isa<FieldDecl>(D)) + if (!RD->isUnion()) ++FieldNo; } @@ -855,14 +869,13 @@ const FieldDecl *CodeGenFunction::FindFlexibleArrayMemberField( static unsigned CountCountedByAttrs(const RecordDecl *RD) { unsigned Num = 0; - for (const Decl *D : RD->decls()) { - if (const auto *FD = dyn_cast<FieldDecl>(D); - FD && FD->hasAttr<CountedByAttr>()) { + for (const FieldDecl *FD : RD->fields()) { + if (FD->getType()->isCountAttributedType()) return ++Num; - } - if (const auto *Rec = dyn_cast<RecordDecl>(D)) - Num += CountCountedByAttrs(Rec); + QualType Ty = FD->getType(); + if (Ty->isRecordType()) + Num += CountCountedByAttrs(Ty->getAsRecordDecl()); } return Num; @@ -925,12 +938,14 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, // Get the flexible array member Decl. const RecordDecl *OuterRD = nullptr; - std::string FAMName; + const FieldDecl *FAMDecl = nullptr; if (const auto *ME = dyn_cast<MemberExpr>(Base)) { // Check if \p Base is referencing the FAM itself. const ValueDecl *VD = ME->getMemberDecl(); OuterRD = VD->getDeclContext()->getOuterLexicalRecordContext(); - FAMName = VD->getNameAsString(); + FAMDecl = dyn_cast<FieldDecl>(VD); + if (!FAMDecl) + return nullptr; } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) { // Check if we're pointing to the whole struct. QualType Ty = DRE->getDecl()->getType(); @@ -955,7 +970,7 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, // }; // }; // - // We don't konw which 'count' to use in this scenario: + // We don't know which 'count' to use in this scenario: // // size_t get_size(struct union_of_fams *p) { // return __builtin_dynamic_object_size(p, 1); @@ -969,12 +984,14 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, if (!OuterRD) return nullptr; + // We call FindFlexibleArrayMemberAndOffset even if FAMDecl is non-null to + // get its offset. uint64_t Offset = 0; - const FieldDecl *FAMDecl = - FindFlexibleArrayMemberField(Ctx, OuterRD, FAMName, Offset); + FAMDecl = + FindFlexibleArrayMemberFieldAndOffset(Ctx, OuterRD, FAMDecl, Offset); Offset = Ctx.toCharUnitsFromBits(Offset).getQuantity(); - if (!FAMDecl || !FAMDecl->hasAttr<CountedByAttr>()) + if (!FAMDecl || !FAMDecl->getType()->isCountAttributedType()) // No flexible array member found or it doesn't have the "counted_by" // attribute. return nullptr; @@ -984,6 +1001,24 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, // Can't find the field referenced by the "counted_by" attribute. return nullptr; + if (isa<DeclRefExpr>(Base)) + // The whole struct is specificed in the __bdos. The calculation of the + // whole size of the structure can be done in two ways: + // + // 1) sizeof(struct S) + count * sizeof(typeof(fam)) + // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam)) + // + // The first will add additional padding after the end of the array, + // allocation while the second method is more precise, but not quite + // expected from programmers. See + // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a + // discussion of the topic. + // + // GCC isn't (currently) able to calculate __bdos on a pointer to the whole + // structure. Therefore, because of the above issue, we'll choose to match + // what GCC does for consistency's sake. + return nullptr; + // Build a load of the counted_by field. bool IsSigned = CountedByFD->getType()->isSignedIntegerType(); Value *CountedByInst = EmitCountedByFieldExpr(Base, FAMDecl, CountedByFD); @@ -1014,32 +1049,9 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type, CharUnits Size = Ctx.getTypeSizeInChars(ArrayTy->getElementType()); llvm::Constant *ElemSize = llvm::ConstantInt::get(ResType, Size.getQuantity(), IsSigned); - Value *FAMSize = + Value *Res = Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned); - FAMSize = Builder.CreateIntCast(FAMSize, ResType, IsSigned); - Value *Res = FAMSize; - - if (isa<DeclRefExpr>(Base)) { - // The whole struct is specificed in the __bdos. - const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(OuterRD); - - // Get the offset of the FAM. - llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned); - Value *OffsetAndFAMSize = - Builder.CreateAdd(FAMOffset, Res, "", !IsSigned, IsSigned); - - // Get the full size of the struct. - llvm::Constant *SizeofStruct = - ConstantInt::get(ResType, Layout.getSize().getQuantity(), IsSigned); - - // max(sizeof(struct s), - // offsetof(struct s, array) + p->count * sizeof(*p->array)) - Res = IsSigned - ? Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax, - OffsetAndFAMSize, SizeofStruct) - : Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax, - OffsetAndFAMSize, SizeofStruct); - } + Res = Builder.CreateIntCast(Res, ResType, IsSigned); // A negative \p IdxInst or \p CountedByInst means that the index lands // outside of the flexible array member. If that's the case, we want to @@ -1129,6 +1141,7 @@ struct BitTest { static BitTest decodeBitTestBuiltin(unsigned BuiltinID); }; + } // namespace BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) { @@ -2115,9 +2128,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction( auto AL = ApplyDebugLocation::CreateArtificial(*this); CharUnits Offset; - Address BufAddr = - Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty, - BufferAlignment); + Address BufAddr = makeNaturalAddressForPointer( + Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy, + BufferAlignment); Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()), Builder.CreateConstByteGEP(BufAddr, Offset++, "summary")); Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()), @@ -2160,7 +2173,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { // Ignore argument 1, the format string. It is not currently used. CallArgList Args; - Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy); + Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy); for (const auto &Item : Layout.Items) { int Size = Item.getSizeByte(); @@ -2200,8 +2213,8 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { if (!isa<Constant>(ArgVal)) { CleanupKind Cleanup = getARCCleanupKind(); QualType Ty = TheExpr->getType(); - Address Alloca = Address::invalid(); - Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); + RawAddress Alloca = RawAddress::invalid(); + RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); ArgVal = EmitARCRetain(Ty, ArgVal); Builder.CreateStore(ArgVal, Addr); pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty, @@ -2234,7 +2247,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction( Layout, BufAddr.getAlignment()); EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args); - return RValue::get(BufAddr.getPointer()); + return RValue::get(BufAddr, *this); } static bool isSpecialUnsignedMultiplySignedResult( @@ -2566,7 +2579,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, if (OP.hasMathErrnoOverride()) ErrnoOverriden = OP.getMathErrnoOverride(); } - // True if 'atttibute__((optnone)) is used. This attibute overrides + // True if 'attribute__((optnone))' is used. This attribute overrides // fast-math which implies math-errno. bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>(); @@ -2596,6 +2609,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_fma: case Builtin::BI__builtin_fmaf: case Builtin::BI__builtin_fmal: + case Builtin::BI__builtin_fmaf16: case Builtin::BIfma: case Builtin::BIfmaf: case Builtin::BIfmal: { @@ -2648,6 +2662,39 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, } if (GenerateIntrinsics) { switch (BuiltinIDIfNoAsmLabel) { + case Builtin::BIacos: + case Builtin::BIacosf: + case Builtin::BIacosl: + case Builtin::BI__builtin_acos: + case Builtin::BI__builtin_acosf: + case Builtin::BI__builtin_acosf16: + case Builtin::BI__builtin_acosl: + case Builtin::BI__builtin_acosf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::acos, Intrinsic::experimental_constrained_acos)); + + case Builtin::BIasin: + case Builtin::BIasinf: + case Builtin::BIasinl: + case Builtin::BI__builtin_asin: + case Builtin::BI__builtin_asinf: + case Builtin::BI__builtin_asinf16: + case Builtin::BI__builtin_asinl: + case Builtin::BI__builtin_asinf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::asin, Intrinsic::experimental_constrained_asin)); + + case Builtin::BIatan: + case Builtin::BIatanf: + case Builtin::BIatanl: + case Builtin::BI__builtin_atan: + case Builtin::BI__builtin_atanf: + case Builtin::BI__builtin_atanf16: + case Builtin::BI__builtin_atanl: + case Builtin::BI__builtin_atanf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::atan, Intrinsic::experimental_constrained_atan)); + case Builtin::BIceil: case Builtin::BIceilf: case Builtin::BIceill: @@ -2668,7 +2715,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_copysignf16: case Builtin::BI__builtin_copysignl: case Builtin::BI__builtin_copysignf128: - return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign)); + return RValue::get( + emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::copysign)); case Builtin::BIcos: case Builtin::BIcosf: @@ -2682,6 +2730,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Intrinsic::cos, Intrinsic::experimental_constrained_cos)); + case Builtin::BIcosh: + case Builtin::BIcoshf: + case Builtin::BIcoshl: + case Builtin::BI__builtin_cosh: + case Builtin::BI__builtin_coshf: + case Builtin::BI__builtin_coshf16: + case Builtin::BI__builtin_coshl: + case Builtin::BI__builtin_coshf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh)); + case Builtin::BIexp: case Builtin::BIexpf: case Builtin::BIexpl: @@ -2713,7 +2772,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // TODO: strictfp support if (Builder.getIsFPConstrained()) break; - return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp10)); + return RValue::get( + emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::exp10)); } case Builtin::BIfabs: case Builtin::BIfabsf: @@ -2723,7 +2783,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_fabsf16: case Builtin::BI__builtin_fabsl: case Builtin::BI__builtin_fabsf128: - return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs)); + return RValue::get( + emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::fabs)); case Builtin::BIfloor: case Builtin::BIfloorf: @@ -2896,6 +2957,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Intrinsic::sin, Intrinsic::experimental_constrained_sin)); + case Builtin::BIsinh: + case Builtin::BIsinhf: + case Builtin::BIsinhl: + case Builtin::BI__builtin_sinh: + case Builtin::BI__builtin_sinhf: + case Builtin::BI__builtin_sinhf16: + case Builtin::BI__builtin_sinhl: + case Builtin::BI__builtin_sinhf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh)); + case Builtin::BIsqrt: case Builtin::BIsqrtf: case Builtin::BIsqrtl: @@ -2910,6 +2982,29 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, SetSqrtFPAccuracy(Call); return RValue::get(Call); } + + case Builtin::BItan: + case Builtin::BItanf: + case Builtin::BItanl: + case Builtin::BI__builtin_tan: + case Builtin::BI__builtin_tanf: + case Builtin::BI__builtin_tanf16: + case Builtin::BI__builtin_tanl: + case Builtin::BI__builtin_tanf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::tan, Intrinsic::experimental_constrained_tan)); + + case Builtin::BItanh: + case Builtin::BItanhf: + case Builtin::BItanhl: + case Builtin::BI__builtin_tanh: + case Builtin::BI__builtin_tanhf: + case Builtin::BI__builtin_tanhf16: + case Builtin::BI__builtin_tanhl: + case Builtin::BI__builtin_tanhf128: + return RValue::get(emitUnaryMaybeConstrainedFPBuiltin( + *this, E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh)); + case Builtin::BItrunc: case Builtin::BItruncf: case Builtin::BItruncl: @@ -2982,7 +3077,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Check NonnullAttribute/NullabilityArg and Alignment. auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg, unsigned ParmNum) { - Value *Val = A.getPointer(); + Value *Val = A.emitRawPointer(*this); EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD, ParmNum); @@ -3011,13 +3106,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_va_end: EmitVAStartEnd(BuiltinID == Builtin::BI__va_start ? EmitScalarExpr(E->getArg(0)) - : EmitVAListRef(E->getArg(0)).getPointer(), + : EmitVAListRef(E->getArg(0)).emitRawPointer(*this), BuiltinID != Builtin::BI__builtin_va_end); return RValue::get(nullptr); case Builtin::BI__builtin_va_copy: { - Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); - Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); - Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), {DstPtr, SrcPtr}); + Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this); + Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this); + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}), + {DstPtr, SrcPtr}); return RValue::get(nullptr); } case Builtin::BIabs: @@ -3127,36 +3223,66 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ctzs: case Builtin::BI__builtin_ctz: case Builtin::BI__builtin_ctzl: - case Builtin::BI__builtin_ctzll: { - Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero); + case Builtin::BI__builtin_ctzll: + case Builtin::BI__builtin_ctzg: { + bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg && + E->getNumArgs() > 1; + + Value *ArgValue = + HasFallback ? EmitScalarExpr(E->getArg(0)) + : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero); llvm::Type *ArgType = ArgValue->getType(); Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); + Value *ZeroUndef = + Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef()); Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) - Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, - "cast"); - return RValue::get(Result); + Result = + Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast"); + if (!HasFallback) + return RValue::get(Result); + + Value *Zero = Constant::getNullValue(ArgType); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); + Value *FallbackValue = EmitScalarExpr(E->getArg(1)); + Value *ResultOrFallback = + Builder.CreateSelect(IsZero, FallbackValue, Result, "ctzg"); + return RValue::get(ResultOrFallback); } case Builtin::BI__builtin_clzs: case Builtin::BI__builtin_clz: case Builtin::BI__builtin_clzl: - case Builtin::BI__builtin_clzll: { - Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero); + case Builtin::BI__builtin_clzll: + case Builtin::BI__builtin_clzg: { + bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg && + E->getNumArgs() > 1; + + Value *ArgValue = + HasFallback ? EmitScalarExpr(E->getArg(0)) + : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero); llvm::Type *ArgType = ArgValue->getType(); Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); + Value *ZeroUndef = + Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef()); Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) - Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, - "cast"); - return RValue::get(Result); + Result = + Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast"); + if (!HasFallback) + return RValue::get(Result); + + Value *Zero = Constant::getNullValue(ArgType); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); + Value *FallbackValue = EmitScalarExpr(E->getArg(1)); + Value *ResultOrFallback = + Builder.CreateSelect(IsZero, FallbackValue, Result, "clzg"); + return RValue::get(ResultOrFallback); } case Builtin::BI__builtin_ffs: case Builtin::BI__builtin_ffsl: @@ -3216,7 +3342,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__popcnt64: case Builtin::BI__builtin_popcount: case Builtin::BI__builtin_popcountl: - case Builtin::BI__builtin_popcountll: { + case Builtin::BI__builtin_popcountll: + case Builtin::BI__builtin_popcountg: { Value *ArgValue = EmitScalarExpr(E->getArg(0)); llvm::Type *ArgType = ArgValue->getType(); @@ -3225,8 +3352,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Type *ResultType = ConvertType(E->getType()); Value *Result = Builder.CreateCall(F, ArgValue); if (Result->getType() != ResultType) - Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, - "cast"); + Result = + Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast"); return RValue::get(Result); } case Builtin::BI__builtin_unpredictable: { @@ -3317,6 +3444,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateAssumption(ConstantInt::getTrue(getLLVMContext()), {OBD}); return RValue::get(nullptr); } + case Builtin::BI__builtin_allow_runtime_check: { + StringRef Kind = + cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts())->getString(); + LLVMContext &Ctx = CGM.getLLVMContext(); + llvm::Value *Allow = Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::allow_runtime_check), + llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind))); + return RValue::get(Allow); + } case Builtin::BI__arithmetic_fence: { // Create the builtin call if FastMath is selected, and the target // supports the builtin, otherwise just return the argument. @@ -3353,13 +3489,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI_byteswap_ushort: case Builtin::BI_byteswap_ulong: case Builtin::BI_byteswap_uint64: { - return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap)); + return RValue::get( + emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bswap)); } case Builtin::BI__builtin_bitreverse8: case Builtin::BI__builtin_bitreverse16: case Builtin::BI__builtin_bitreverse32: case Builtin::BI__builtin_bitreverse64: { - return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse)); + return RValue::get( + emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bitreverse)); } case Builtin::BI__builtin_rotateleft8: case Builtin::BI__builtin_rotateleft16: @@ -3443,6 +3581,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); return RValue::get(Builder.CreateCall(F)); } + case Builtin::BI__builtin_readsteadycounter: { + Function *F = CGM.getIntrinsic(Intrinsic::readsteadycounter); + return RValue::get(Builder.CreateCall(F)); + } case Builtin::BI__builtin___clear_cache: { Value *Begin = EmitScalarExpr(E->getArg(0)); Value *End = EmitScalarExpr(E->getArg(1)); @@ -3452,6 +3594,18 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_trap: EmitTrapCall(Intrinsic::trap); return RValue::get(nullptr); + case Builtin::BI__builtin_verbose_trap: { + llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation(); + if (getDebugInfo()) { + TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor( + TrapLocation, *E->getArg(0)->tryEvaluateString(getContext()), + *E->getArg(1)->tryEvaluateString(getContext())); + } + ApplyDebugLocation ApplyTrapDI(*this, TrapLocation); + // Currently no attempt is made to prevent traps from being merged. + EmitTrapCall(Intrinsic::trap); + return RValue::get(nullptr); + } case Builtin::BI__debugbreak: EmitTrapCall(Intrinsic::debugtrap); return RValue::get(nullptr); @@ -3489,7 +3643,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // frexpl instead of legalizing this type in the BE. if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble()) break; - LLVM_FALLTHROUGH; + [[fallthrough]]; } case Builtin::BI__builtin_frexp: case Builtin::BI__builtin_frexpf: @@ -3632,67 +3786,90 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Intrinsic::abs, EmitScalarExpr(E->getArg(0)), Builder.getFalse(), nullptr, "elt.abs"); else - Result = emitUnaryBuiltin(*this, E, llvm::Intrinsic::fabs, "elt.abs"); + Result = emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::fabs, "elt.abs"); return RValue::get(Result); } - + case Builtin::BI__builtin_elementwise_acos: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::acos, "elt.acos")); + case Builtin::BI__builtin_elementwise_asin: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::asin, "elt.asin")); + case Builtin::BI__builtin_elementwise_atan: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::atan, "elt.atan")); case Builtin::BI__builtin_elementwise_ceil: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::ceil, "elt.ceil")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::ceil, "elt.ceil")); case Builtin::BI__builtin_elementwise_exp: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::exp, "elt.exp")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::exp, "elt.exp")); case Builtin::BI__builtin_elementwise_exp2: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::exp2, "elt.exp2")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::exp2, "elt.exp2")); case Builtin::BI__builtin_elementwise_log: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::log, "elt.log")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::log, "elt.log")); case Builtin::BI__builtin_elementwise_log2: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::log2, "elt.log2")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::log2, "elt.log2")); case Builtin::BI__builtin_elementwise_log10: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::log10, "elt.log10")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::log10, "elt.log10")); case Builtin::BI__builtin_elementwise_pow: { - return RValue::get(emitBinaryBuiltin(*this, E, llvm::Intrinsic::pow)); + return RValue::get( + emitBuiltinWithOneOverloadedType<2>(*this, E, llvm::Intrinsic::pow)); } case Builtin::BI__builtin_elementwise_bitreverse: - return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::bitreverse, - "elt.bitreverse")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::bitreverse, "elt.bitreverse")); case Builtin::BI__builtin_elementwise_cos: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::cos, "elt.cos")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::cos, "elt.cos")); + case Builtin::BI__builtin_elementwise_cosh: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::cosh, "elt.cosh")); case Builtin::BI__builtin_elementwise_floor: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::floor, "elt.floor")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::floor, "elt.floor")); case Builtin::BI__builtin_elementwise_roundeven: - return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::roundeven, - "elt.roundeven")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::roundeven, "elt.roundeven")); case Builtin::BI__builtin_elementwise_round: - return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::round, - "elt.round")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::round, "elt.round")); case Builtin::BI__builtin_elementwise_rint: - return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::rint, - "elt.rint")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::rint, "elt.rint")); case Builtin::BI__builtin_elementwise_nearbyint: - return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::nearbyint, - "elt.nearbyint")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::nearbyint, "elt.nearbyint")); case Builtin::BI__builtin_elementwise_sin: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::sin, "elt.sin")); - + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::sin, "elt.sin")); + case Builtin::BI__builtin_elementwise_sinh: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::sinh, "elt.sinh")); + case Builtin::BI__builtin_elementwise_tan: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::tan, "elt.tan")); + case Builtin::BI__builtin_elementwise_tanh: + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::tanh, "elt.tanh")); case Builtin::BI__builtin_elementwise_trunc: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::trunc, "elt.trunc")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::trunc, "elt.trunc")); case Builtin::BI__builtin_elementwise_canonicalize: - return RValue::get( - emitUnaryBuiltin(*this, E, llvm::Intrinsic::canonicalize, "elt.canonicalize")); + return RValue::get(emitBuiltinWithOneOverloadedType<1>( + *this, E, llvm::Intrinsic::canonicalize, "elt.canonicalize")); case Builtin::BI__builtin_elementwise_copysign: - return RValue::get(emitBinaryBuiltin(*this, E, llvm::Intrinsic::copysign)); + return RValue::get(emitBuiltinWithOneOverloadedType<2>( + *this, E, llvm::Intrinsic::copysign)); case Builtin::BI__builtin_elementwise_fma: - return RValue::get(emitTernaryBuiltin(*this, E, llvm::Intrinsic::fma)); + return RValue::get( + emitBuiltinWithOneOverloadedType<3>(*this, E, llvm::Intrinsic::fma)); case Builtin::BI__builtin_elementwise_add_sat: case Builtin::BI__builtin_elementwise_sub_sat: { Value *Op0 = EmitScalarExpr(E->getArg(0)); @@ -3746,9 +3923,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, } case Builtin::BI__builtin_reduce_max: { - auto GetIntrinsicID = [](QualType QT) { + auto GetIntrinsicID = [this](QualType QT) { if (auto *VecTy = QT->getAs<VectorType>()) QT = VecTy->getElementType(); + else if (QT->isSizelessVectorType()) + QT = QT->getSizelessVectorEltType(CGM.getContext()); + if (QT->isSignedIntegerType()) return llvm::Intrinsic::vector_reduce_smax; if (QT->isUnsignedIntegerType()) @@ -3756,14 +3936,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, assert(QT->isFloatingType() && "must have a float here"); return llvm::Intrinsic::vector_reduce_fmax; }; - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min")); } case Builtin::BI__builtin_reduce_min: { - auto GetIntrinsicID = [](QualType QT) { + auto GetIntrinsicID = [this](QualType QT) { if (auto *VecTy = QT->getAs<VectorType>()) QT = VecTy->getElementType(); + else if (QT->isSizelessVectorType()) + QT = QT->getSizelessVectorEltType(CGM.getContext()); + if (QT->isSignedIntegerType()) return llvm::Intrinsic::vector_reduce_smin; if (QT->isUnsignedIntegerType()) @@ -3772,24 +3955,24 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return llvm::Intrinsic::vector_reduce_fmin; }; - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min")); } case Builtin::BI__builtin_reduce_add: - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, llvm::Intrinsic::vector_reduce_add, "rdx.add")); case Builtin::BI__builtin_reduce_mul: - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, llvm::Intrinsic::vector_reduce_mul, "rdx.mul")); case Builtin::BI__builtin_reduce_xor: - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, llvm::Intrinsic::vector_reduce_xor, "rdx.xor")); case Builtin::BI__builtin_reduce_or: - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, llvm::Intrinsic::vector_reduce_or, "rdx.or")); case Builtin::BI__builtin_reduce_and: - return RValue::get(emitUnaryBuiltin( + return RValue::get(emitBuiltinWithOneOverloadedType<1>( *this, E, llvm::Intrinsic::vector_reduce_and, "rdx.and")); case Builtin::BI__builtin_matrix_transpose: { @@ -3811,13 +3994,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); Address Src = EmitPointerWithAlignment(E->getArg(0)); - EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); Value *Result = MB.CreateColumnMajorLoad( - Src.getElementType(), Src.getPointer(), + Src.getElementType(), Src.emitRawPointer(*this), Align(Src.getAlignment().getQuantity()), Stride, IsVolatile, - ResultTy->getNumRows(), ResultTy->getNumColumns(), - "matrix"); + ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix"); return RValue::get(Result); } @@ -3832,11 +4015,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, assert(PtrTy && "arg1 must be of pointer type"); bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); - EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(), - E->getArg(1)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dst.emitRawPointer(*this)), + E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, + 0); Value *Result = MB.CreateColumnMajorStore( - Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()), - Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns()); + Matrix, Dst.emitRawPointer(*this), + Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile, + MatrixTy->getNumRows(), MatrixTy->getNumColumns()); return RValue::get(Result); } @@ -3995,7 +4180,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_bzero: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *SizeVal = EmitScalarExpr(E->getArg(1)); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); return RValue::get(nullptr); @@ -4006,12 +4191,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(0)); Address Dest = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(1)->getType(), - E->getArg(1)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); + EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), + E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, + 0); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(nullptr); } case Builtin::BImemcpy: @@ -4026,10 +4213,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateMemCpy(Dest, Src, SizeVal, false); if (BuiltinID == Builtin::BImempcpy || BuiltinID == Builtin::BI__builtin_mempcpy) - return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(), - Dest.getPointer(), SizeVal)); + return RValue::get(Builder.CreateInBoundsGEP( + Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal)); else - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_memcpy_inline: { @@ -4061,7 +4248,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemCpy(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_objc_memmove_collectable: { @@ -4070,7 +4257,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *SizeVal = EmitScalarExpr(E->getArg(2)); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestAddr, SrcAddr, SizeVal); - return RValue::get(DestAddr.getPointer()); + return RValue::get(DestAddr, *this); } case Builtin::BI__builtin___memmove_chk: { @@ -4087,7 +4274,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BImemmove: @@ -4098,7 +4285,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0); EmitArgCheck(TCK_Load, Src, E->getArg(1), 1); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BImemset: case Builtin::BI__builtin_memset: { @@ -4106,10 +4293,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_memset_inline: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); @@ -4117,8 +4304,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); uint64_t Size = E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue(); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); Builder.CreateMemSetInline(Dest, ByteVal, Size); return RValue::get(nullptr); } @@ -4137,7 +4325,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.getInt8Ty()); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_wmemchr: { // The MSVC runtime library does not provide a definition of wmemchr, so we @@ -4359,14 +4547,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Store the stack pointer to the setjmp buffer. Value *StackAddr = Builder.CreateStackSave(); - assert(Buf.getPointer()->getType() == StackAddr->getType()); + assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType()); Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2); Builder.CreateStore(StackAddr, StackSaveSlot); // Call LLVM's EH setjmp, which is lightweight. Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); - return RValue::get(Builder.CreateCall(F, Buf.getPointer())); + return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this))); } case Builtin::BI__builtin_longjmp: { Value *Buf = EmitScalarExpr(E->getArg(0)); @@ -5202,6 +5390,76 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__iso_volatile_store64: return RValue::get(EmitISOVolatileStore(*this, E)); + case Builtin::BI__builtin_ptrauth_sign_constant: + return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType())); + + case Builtin::BI__builtin_ptrauth_auth: + case Builtin::BI__builtin_ptrauth_auth_and_resign: + case Builtin::BI__builtin_ptrauth_blend_discriminator: + case Builtin::BI__builtin_ptrauth_sign_generic_data: + case Builtin::BI__builtin_ptrauth_sign_unauthenticated: + case Builtin::BI__builtin_ptrauth_strip: { + // Emit the arguments. + SmallVector<llvm::Value *, 5> Args; + for (auto argExpr : E->arguments()) + Args.push_back(EmitScalarExpr(argExpr)); + + // Cast the value to intptr_t, saving its original type. + llvm::Type *OrigValueType = Args[0]->getType(); + if (OrigValueType->isPointerTy()) + Args[0] = Builder.CreatePtrToInt(Args[0], IntPtrTy); + + switch (BuiltinID) { + case Builtin::BI__builtin_ptrauth_auth_and_resign: + if (Args[4]->getType()->isPointerTy()) + Args[4] = Builder.CreatePtrToInt(Args[4], IntPtrTy); + [[fallthrough]]; + + case Builtin::BI__builtin_ptrauth_auth: + case Builtin::BI__builtin_ptrauth_sign_unauthenticated: + if (Args[2]->getType()->isPointerTy()) + Args[2] = Builder.CreatePtrToInt(Args[2], IntPtrTy); + break; + + case Builtin::BI__builtin_ptrauth_sign_generic_data: + if (Args[1]->getType()->isPointerTy()) + Args[1] = Builder.CreatePtrToInt(Args[1], IntPtrTy); + break; + + case Builtin::BI__builtin_ptrauth_blend_discriminator: + case Builtin::BI__builtin_ptrauth_strip: + break; + } + + // Call the intrinsic. + auto IntrinsicID = [&]() -> unsigned { + switch (BuiltinID) { + case Builtin::BI__builtin_ptrauth_auth: + return llvm::Intrinsic::ptrauth_auth; + case Builtin::BI__builtin_ptrauth_auth_and_resign: + return llvm::Intrinsic::ptrauth_resign; + case Builtin::BI__builtin_ptrauth_blend_discriminator: + return llvm::Intrinsic::ptrauth_blend; + case Builtin::BI__builtin_ptrauth_sign_generic_data: + return llvm::Intrinsic::ptrauth_sign_generic; + case Builtin::BI__builtin_ptrauth_sign_unauthenticated: + return llvm::Intrinsic::ptrauth_sign; + case Builtin::BI__builtin_ptrauth_strip: + return llvm::Intrinsic::ptrauth_strip; + } + llvm_unreachable("bad ptrauth intrinsic"); + }(); + auto Intrinsic = CGM.getIntrinsic(IntrinsicID); + llvm::Value *Result = EmitRuntimeCall(Intrinsic, Args); + + if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data && + BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator && + OrigValueType->isPointerTy()) { + Result = Builder.CreateIntToPtr(Result, OrigValueType); + } + return RValue::get(Result); + } + case Builtin::BI__exception_code: case Builtin::BI_exception_code: return RValue::get(EmitSEHExceptionCode()); @@ -5454,7 +5712,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, } // OpenCL v2.0, s6.13.17 - Enqueue kernel function. - // It contains four different overload formats specified in Table 6.13.17.1. + // Table 6.13.17.1 specifies four overload forms of enqueue_kernel. + // The code below expands the builtin call to a call to one of the following + // functions that an OpenCL runtime library will have to provide: + // __enqueue_kernel_basic + // __enqueue_kernel_varargs + // __enqueue_kernel_basic_events + // __enqueue_kernel_events_varargs case Builtin::BIenqueue_kernel: { StringRef Name; // Generated function call name unsigned NumArgs = E->getNumArgs(); @@ -5466,8 +5730,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Value *Queue = EmitScalarExpr(E->getArg(0)); llvm::Value *Flags = EmitScalarExpr(E->getArg(1)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(2)); - llvm::Value *Range = NDRangeL.getAddress(*this).getPointer(); - llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType(); + llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(*this); + llvm::Type *RangeTy = NDRangeL.getAddress().getType(); if (NumArgs == 4) { // The most basic form of the call with parameters: @@ -5486,7 +5750,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy); AttrBuilder B(Builder.getContext()); - B.addByValAttr(NDRangeL.getAddress(*this).getElementType()); + B.addByValAttr(NDRangeL.getAddress().getElementType()); llvm::AttributeList ByValAttrSet = llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B); @@ -5575,9 +5839,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, getContext(), Expr::NPC_ValueDependentIsNotNull)) { EventWaitList = llvm::ConstantPointerNull::get(PtrTy); } else { - EventWaitList = E->getArg(4)->getType()->isArrayType() - ? EmitArrayToPointerDecay(E->getArg(4)).getPointer() - : EmitScalarExpr(E->getArg(4)); + EventWaitList = + E->getArg(4)->getType()->isArrayType() + ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this) + : EmitScalarExpr(E->getArg(4)); // Convert to generic address space. EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy); } @@ -5634,7 +5899,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitLifetimeEnd(TmpSize, TmpPtr); return Call; } - [[fallthrough]]; + llvm_unreachable("Unexpected enqueue_kernel signature"); } // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block // parameter. @@ -5673,7 +5938,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Type *GenericVoidPtrTy = Builder.getPtrTy( getContext().getTargetAddressSpace(LangAS::opencl_generic)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(0)); - llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer(); + llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(*this); auto Info = CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1)); Value *Kernel = @@ -5691,7 +5956,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Name), {NDRange, Kernel, Block})); } - case Builtin::BI__builtin_store_half: case Builtin::BI__builtin_store_halff: { Value *Val = EmitScalarExpr(E->getArg(0)); @@ -5710,14 +5974,19 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *HalfVal = Builder.CreateLoad(Address); return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy())); } + case Builtin::BI__builtin_printf: case Builtin::BIprintf: if (getTarget().getTriple().isNVPTX() || - getTarget().getTriple().isAMDGCN()) { + getTarget().getTriple().isAMDGCN() || + (getTarget().getTriple().isSPIRV() && + getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) { if (getLangOpts().OpenMPIsTargetDevice) return EmitOpenMPDevicePrintfCallExpr(E); if (getTarget().getTriple().isNVPTX()) return EmitNVPTXDevicePrintfCallExpr(E); - if (getTarget().getTriple().isAMDGCN() && getLangOpts().HIP) + if ((getTarget().getTriple().isAMDGCN() || + getTarget().getTriple().isSPIRV()) && + getLangOpts().HIP) return EmitAMDGPUDevicePrintfCallExpr(E); } @@ -5726,7 +5995,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_canonicalizef: case Builtin::BI__builtin_canonicalizef16: case Builtin::BI__builtin_canonicalizel: - return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize)); + return RValue::get( + emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::canonicalize)); case Builtin::BI__builtin_thread_pointer: { if (!getContext().getTargetInfo().isTLSSupported()) @@ -5757,7 +6027,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy0 = FTy->getParamType(0); if (PTy0 != Arg0Val->getType()) { if (Arg0Ty->isArrayType()) - Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer(); + Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*this); else Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0); } @@ -5795,7 +6065,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy1 = FTy->getParamType(1); if (PTy1 != Arg1Val->getType()) { if (Arg1Ty->isArrayType()) - Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer(); + Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*this); else Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1); } @@ -5809,7 +6079,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ms_va_start: case Builtin::BI__builtin_ms_va_end: return RValue::get( - EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), + EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this), BuiltinID == Builtin::BI__builtin_ms_va_start)); case Builtin::BI__builtin_ms_va_copy: { @@ -5833,11 +6103,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto Name = CGM.getCUDARuntime().getDeviceSideName( cast<DeclRefExpr>(E->getArg(0)->IgnoreImpCasts())->getDecl()); auto Str = CGM.GetAddrOfConstantCString(Name, ""); - llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), - llvm::ConstantInt::get(SizeTy, 0)}; - auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(), - Str.getPointer(), Zeros); - return RValue::get(Ptr); + return RValue::get(Str.getPointer()); } } @@ -5851,8 +6117,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // If this is a predefined lib function (e.g. malloc), emit the call // using exactly the normal call path. if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) - return emitLibraryCall(*this, FD, E, - cast<llvm::Constant>(EmitScalarExpr(E->getCallee()))); + return emitLibraryCall(*this, FD, E, CGM.getRawFunctionPointer(FD)); // Check that a call to a target specific builtin has the correct target // features. @@ -5871,6 +6136,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch()); if (!Prefix.empty()) { IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name); + if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" && + getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA) + IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin("amdgcn", Name); // NOTE we don't need to perform a compatibility flag check here since the // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. @@ -5907,8 +6175,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, } } - assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && - "Must be able to losslessly bit cast to param"); // Cast vector type (e.g., v256i32) to x86_amx, this only happen // in amx intrinsics. if (PTy->isX86_AMXTy()) @@ -5938,8 +6204,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, } } - assert(V->getType()->canLosslesslyBitCastTo(RetTy) && - "Must be able to losslessly bit cast result type"); // Cast x86_amx to vector type (e.g., v256i32), this only happen // in amx intrinsics. if (V->getType()->isX86_AMXTy()) @@ -5973,7 +6237,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return RValue::get(nullptr); return RValue::get(V); case TEK_Aggregate: - return RValue::getAggregate(ReturnValue.getValue(), + return RValue::getAggregate(ReturnValue.getAddress(), ReturnValue.isVolatile()); case TEK_Complex: llvm_unreachable("No current target builtin returns complex"); @@ -5981,6 +6245,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr"); } + // EmitHLSLBuiltinExpr will check getLangOpts().HLSL + if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E)) + return RValue::get(V); + if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice) return EmitHipStdParUnsupportedBuiltin(this, FD); @@ -6041,6 +6309,10 @@ static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, case llvm::Triple::riscv32: case llvm::Triple::riscv64: return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue); + case llvm::Triple::spirv64: + if (CGF->getTarget().getTriple().getOS() != llvm::Triple::OSType::AMDHSA) + return nullptr; + return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); default: return nullptr; } @@ -7080,8 +7352,6 @@ static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = { { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, }, { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, }, { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, }, - { NEON::BI__builtin_neon_vbsl_f16, NEON::BI__builtin_neon_vbsl_v, }, - { NEON::BI__builtin_neon_vbslq_f16, NEON::BI__builtin_neon_vbslq_v, }, { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, }, { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, }, { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, }, @@ -7100,8 +7370,6 @@ static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = { { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, }, { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, }, { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, }, - { NEON::BI__builtin_neon_vext_f16, NEON::BI__builtin_neon_vext_v, }, - { NEON::BI__builtin_neon_vextq_f16, NEON::BI__builtin_neon_vextq_v, }, { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, }, { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, }, { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, }, @@ -7204,12 +7472,6 @@ static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = { { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v }, { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v }, { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v }, - { NEON::BI__builtin_neon_vtrn_f16, NEON::BI__builtin_neon_vtrn_v, }, - { NEON::BI__builtin_neon_vtrnq_f16, NEON::BI__builtin_neon_vtrnq_v, }, - { NEON::BI__builtin_neon_vuzp_f16, NEON::BI__builtin_neon_vuzp_v, }, - { NEON::BI__builtin_neon_vuzpq_f16, NEON::BI__builtin_neon_vuzpq_v, }, - { NEON::BI__builtin_neon_vzip_f16, NEON::BI__builtin_neon_vzip_v, }, - { NEON::BI__builtin_neon_vzipq_f16, NEON::BI__builtin_neon_vzipq_v, }, // The mangling rules cause us to have one ID for each type for vldap1(q)_lane // and vstl1(q)_lane, but codegen is equivalent for all of them. Choose an // arbitrary one to be handled as tha canonical variation. @@ -8739,7 +9001,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.getPointer()); + Ops.push_back(PtrOp0.emitRawPointer(*this)); continue; } } @@ -8766,7 +9028,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); - Ops.push_back(PtrOp1.getPointer()); + Ops.push_back(PtrOp1.emitRawPointer(*this)); continue; } } @@ -9187,7 +9449,7 @@ Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID, if (ReturnValue.isNull()) return MvecOut; else - return Builder.CreateStore(MvecOut, ReturnValue.getValue()); + return Builder.CreateStore(MvecOut, ReturnValue.getAddress()); } case CustomCodeGen::VST24: { @@ -10077,11 +10339,15 @@ Value *CodeGenFunction::EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *Ty, ArrayRef<Value *> Ops) { assert((TypeFlags.isTupleSet() || TypeFlags.isTupleGet()) && - "Expects TypleFlag isTupleSet or TypeFlags.isTupleSet()"); + "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()"); unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue(); auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>( - TypeFlags.isTupleSet() ? Ops[2]->getType() : Ty); + TypeFlags.isTupleSet() ? Ops[2]->getType() : Ty); + + if (!SingleVecTy) + return nullptr; + Value *Idx = ConstantInt::get(CGM.Int64Ty, I * SingleVecTy->getMinNumElements()); @@ -10096,6 +10362,10 @@ Value *CodeGenFunction::EmitSVETupleCreate(const SVETypeFlags &TypeFlags, assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate"); auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType()); + + if (!SrcTy) + return nullptr; + unsigned MinElts = SrcTy->getMinNumElements(); Value *Call = llvm::PoisonValue::get(Ty); for (unsigned I = 0; I < Ops.size(); I++) { @@ -10637,6 +10907,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, BuiltinID <= clang::AArch64::LastSMEBuiltin) return EmitAArch64SMEBuiltinExpr(BuiltinID, E); + if (BuiltinID == Builtin::BI__builtin_cpu_supports) + return EmitAArch64CpuSupports(E); + unsigned HintID = static_cast<unsigned>(-1); switch (BuiltinID) { default: break; @@ -10670,16 +10943,20 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); } + if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) { + Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break); + llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall(F, Builder.CreateZExt(Arg, CGM.Int32Ty)); + } + if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) { // Create call to __arm_sme_state and store the results to the two pointers. CallInst *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction( llvm::FunctionType::get(StructType::get(CGM.Int64Ty, CGM.Int64Ty), {}, false), "__arm_sme_state")); - auto Attrs = - AttributeList() - .addFnAttribute(getLLVMContext(), "aarch64_pstate_sm_compatible") - .addFnAttribute(getLLVMContext(), "aarch64_pstate_za_preserved"); + auto Attrs = AttributeList().addFnAttribute(getLLVMContext(), + "aarch64_pstate_sm_compatible"); CI->setAttributes(Attrs); CI->setCallingConv( llvm::CallingConv:: @@ -11318,6 +11595,15 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F, {Address, RW, Locality, Data}); } + if (BuiltinID == AArch64::BI__hlt) { + Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hlt); + Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))}); + + // Return 0 for convenience, even though MSVC returns some other undefined + // value. + return ConstantInt::get(Builder.getInt32Ty(), 0); + } + // Handle MSVC intrinsics before argument evaluation to prevent double // evaluation. if (std::optional<MSVCIntrin> MsvcIntId = @@ -11360,7 +11646,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.getPointer()); + Ops.push_back(PtrOp0.emitRawPointer(*this)); continue; } } @@ -12044,7 +12330,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, "vgetq_lane"); } - case clang::AArch64::BI_InterlockedAdd: { + case clang::AArch64::BI_InterlockedAdd: + case clang::AArch64::BI_InterlockedAdd64: { Address DestAddr = CheckAtomicAlignment(*this, E); Value *Val = EmitScalarExpr(E->getArg(1)); AtomicRMWInst *RMWI = @@ -13225,15 +13512,15 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID, if (!getDebugInfo()) { CGM.Error(E->getExprLoc(), "using __builtin_preserve_field_info() without -g"); - return IsBitField ? EmitLValue(Arg).getBitFieldPointer() - : EmitLValue(Arg).getPointer(*this); + return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) + : EmitLValue(Arg).emitRawPointer(*this); } // Enable underlying preserve_*_access_index() generation. bool OldIsInPreservedAIRegion = IsInPreservedAIRegion; IsInPreservedAIRegion = true; - Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer() - : EmitLValue(Arg).getPointer(*this); + Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) + : EmitLValue(Arg).emitRawPointer(*this); IsInPreservedAIRegion = OldIsInPreservedAIRegion; ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); @@ -13931,7 +14218,7 @@ Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) { // Grab the appropriate field from __cpu_model. llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, Index)}; - llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs); + llvm::Value *CpuValue = Builder.CreateInBoundsGEP(STy, CpuModel, Idxs); CpuValue = Builder.CreateAlignedLoad(Int32Ty, CpuValue, CharUnits::fromQuantity(4)); @@ -13943,6 +14230,8 @@ Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) { Value *CodeGenFunction::EmitX86CpuSupports(const CallExpr *E) { const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts(); StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString(); + if (!getContext().getTargetInfo().validateCpuSupports(FeatureStr)) + return Builder.getFalse(); return EmitX86CpuSupports(FeatureStr); } @@ -13971,7 +14260,7 @@ CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) { // global in the struct STy. Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3), Builder.getInt32(0)}; - Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); + Value *CpuFeatures = Builder.CreateInBoundsGEP(STy, CpuModel, Idxs); Value *Features = Builder.CreateAlignedLoad(Int32Ty, CpuFeatures, CharUnits::fromQuantity(4)); @@ -13992,7 +14281,7 @@ CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) { continue; Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(i - 1)}; Value *Features = Builder.CreateAlignedLoad( - Int32Ty, Builder.CreateGEP(ATy, CpuFeatures2, Idxs), + Int32Ty, Builder.CreateInBoundsGEP(ATy, CpuFeatures2, Idxs), CharUnits::fromQuantity(4)); // Check the value of the bit corresponding to the feature requested. Value *Mask = Builder.getInt32(M); @@ -14025,6 +14314,21 @@ Value *CodeGenFunction::EmitX86CpuInit() { return Builder.CreateCall(Func); } +Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) { + const Expr *ArgExpr = E->getArg(0)->IgnoreParenCasts(); + StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString(); + llvm::SmallVector<StringRef, 8> Features; + ArgStr.split(Features, "+"); + for (auto &Feature : Features) { + Feature = Feature.trim(); + if (!llvm::AArch64::parseFMVExtension(Feature)) + return Builder.getFalse(); + if (Feature != "default") + Features.push_back(Feature); + } + return EmitAArch64CpuSupports(Features); +} + llvm::Value * CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) { uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs); @@ -14053,11 +14357,11 @@ CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) { Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E) { - if (BuiltinID == X86::BI__builtin_cpu_is) + if (BuiltinID == Builtin::BI__builtin_cpu_is) return EmitX86CpuIs(E); - if (BuiltinID == X86::BI__builtin_cpu_supports) + if (BuiltinID == Builtin::BI__builtin_cpu_supports) return EmitX86CpuSupports(E); - if (BuiltinID == X86::BI__builtin_cpu_init) + if (BuiltinID == Builtin::BI__builtin_cpu_init) return EmitX86CpuInit(); // Handle MSVC intrinsics before argument evaluation to prevent double @@ -14208,14 +14512,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } case X86::BI_mm_setcsr: case X86::BI__builtin_ia32_ldmxcsr: { - Address Tmp = CreateMemTemp(E->getArg(0)->getType()); + RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType()); Builder.CreateStore(Ops[0], Tmp); return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), Tmp.getPointer()); } case X86::BI_mm_getcsr: case X86::BI__builtin_ia32_stmxcsr: { - Address Tmp = CreateMemTemp(E->getType()); + RawAddress Tmp = CreateMemTemp(E->getType()); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), Tmp.getPointer()); return Builder.CreateLoad(Tmp, "stmxcsr"); @@ -15753,14 +16057,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F, {Ops[0]}); } - // 3DNow! - case X86::BI__builtin_ia32_pswapdsf: - case X86::BI__builtin_ia32_pswapdsi: { - llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); - Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); - return Builder.CreateCall(F, Ops, "pswapd"); - } case X86::BI__builtin_ia32_rdrand16_step: case X86::BI__builtin_ia32_rdrand32_step: case X86::BI__builtin_ia32_rdrand64_step: @@ -16542,9 +16838,144 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Intrinsic::ID ID = Intrinsic::not_intrinsic; +#include "llvm/TargetParser/PPCTargetParser.def" + auto GenAIXPPCBuiltinCpuExpr = [&](unsigned SupportMethod, unsigned FieldIdx, + unsigned Mask, CmpInst::Predicate CompOp, + unsigned OpValue) -> Value * { + if (SupportMethod == BUILTIN_PPC_FALSE) + return llvm::ConstantInt::getFalse(ConvertType(E->getType())); + + if (SupportMethod == BUILTIN_PPC_TRUE) + return llvm::ConstantInt::getTrue(ConvertType(E->getType())); + + assert(SupportMethod <= SYS_CALL && "Invalid value for SupportMethod."); + + llvm::Value *FieldValue = nullptr; + if (SupportMethod == USE_SYS_CONF) { + llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE); + llvm::Constant *SysConf = + CGM.CreateRuntimeVariable(STy, "_system_configuration"); + + // Grab the appropriate field from _system_configuration. + llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0), + ConstantInt::get(Int32Ty, FieldIdx)}; + + FieldValue = Builder.CreateInBoundsGEP(STy, SysConf, Idxs); + FieldValue = Builder.CreateAlignedLoad(Int32Ty, FieldValue, + CharUnits::fromQuantity(4)); + } else if (SupportMethod == SYS_CALL) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(Int64Ty, Int32Ty, false); + llvm::FunctionCallee Func = + CGM.CreateRuntimeFunction(FTy, "getsystemcfg"); + + FieldValue = + Builder.CreateCall(Func, {ConstantInt::get(Int32Ty, FieldIdx)}); + } + assert(FieldValue && + "SupportMethod value is not defined in PPCTargetParser.def."); + + if (Mask) + FieldValue = Builder.CreateAnd(FieldValue, Mask); + + llvm::Type *ValueType = FieldValue->getType(); + bool IsValueType64Bit = ValueType->isIntegerTy(64); + assert( + (IsValueType64Bit || ValueType->isIntegerTy(32)) && + "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr()."); + + return Builder.CreateICmp( + CompOp, FieldValue, + ConstantInt::get(IsValueType64Bit ? Int64Ty : Int32Ty, OpValue)); + }; + switch (BuiltinID) { default: return nullptr; + case Builtin::BI__builtin_cpu_is: { + const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts(); + StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString(); + llvm::Triple Triple = getTarget().getTriple(); + + unsigned LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue; + typedef std::tuple<unsigned, unsigned, unsigned, unsigned> CPUInfo; + + std::tie(LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue) = + static_cast<CPUInfo>(StringSwitch<CPUInfo>(CPUStr) +#define PPC_CPU(NAME, Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, \ + AIXID) \ + .Case(NAME, {Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, AIXID}) +#include "llvm/TargetParser/PPCTargetParser.def" + .Default({BUILTIN_PPC_UNSUPPORTED, 0, + BUILTIN_PPC_UNSUPPORTED, 0})); + + if (Triple.isOSAIX()) { + assert((AIXSupportMethod != BUILTIN_PPC_UNSUPPORTED) && + "Invalid CPU name. Missed by SemaChecking?"); + return GenAIXPPCBuiltinCpuExpr(AIXSupportMethod, AIX_SYSCON_IMPL_IDX, 0, + ICmpInst::ICMP_EQ, AIXIDValue); + } + + assert(Triple.isOSLinux() && + "__builtin_cpu_is() is only supported for AIX and Linux."); + + assert((LinuxSupportMethod != BUILTIN_PPC_UNSUPPORTED) && + "Invalid CPU name. Missed by SemaChecking?"); + + if (LinuxSupportMethod == BUILTIN_PPC_FALSE) + return llvm::ConstantInt::getFalse(ConvertType(E->getType())); + + Value *Op0 = llvm::ConstantInt::get(Int32Ty, PPC_FAWORD_CPUID); + llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_fixed_addr_ld); + Value *TheCall = Builder.CreateCall(F, {Op0}, "cpu_is"); + return Builder.CreateICmpEQ(TheCall, + llvm::ConstantInt::get(Int32Ty, LinuxIDValue)); + } + case Builtin::BI__builtin_cpu_supports: { + llvm::Triple Triple = getTarget().getTriple(); + const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts(); + StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString(); + if (Triple.isOSAIX()) { + unsigned SupportMethod, FieldIdx, Mask, Value; + CmpInst::Predicate CompOp; + typedef std::tuple<unsigned, unsigned, unsigned, CmpInst::Predicate, + unsigned> + CPUSupportType; + std::tie(SupportMethod, FieldIdx, Mask, CompOp, Value) = + static_cast<CPUSupportType>(StringSwitch<CPUSupportType>(CPUStr) +#define PPC_AIX_FEATURE(NAME, DESC, SUPPORT_METHOD, INDEX, MASK, COMP_OP, \ + VALUE) \ + .Case(NAME, {SUPPORT_METHOD, INDEX, MASK, COMP_OP, VALUE}) +#include "llvm/TargetParser/PPCTargetParser.def" + .Default({BUILTIN_PPC_FALSE, 0, 0, + CmpInst::Predicate(), 0})); + return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, Mask, CompOp, + Value); + } + + assert(Triple.isOSLinux() && + "__builtin_cpu_supports() is only supported for AIX and Linux."); + unsigned FeatureWord; + unsigned BitMask; + std::tie(FeatureWord, BitMask) = + StringSwitch<std::pair<unsigned, unsigned>>(CPUStr) +#define PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \ + .Case(Name, {FA_WORD, Bitmask}) +#include "llvm/TargetParser/PPCTargetParser.def" + .Default({0, 0}); + if (!BitMask) + return Builder.getFalse(); + Value *Op0 = llvm::ConstantInt::get(Int32Ty, FeatureWord); + llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_fixed_addr_ld); + Value *TheCall = Builder.CreateCall(F, {Op0}, "cpu_supports"); + Value *Mask = + Builder.CreateAnd(TheCall, llvm::ConstantInt::get(Int32Ty, BitMask)); + return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(Int32Ty)); +#undef PPC_FAWORD_HWCAP +#undef PPC_FAWORD_HWCAP2 +#undef PPC_FAWORD_CPUID + } + // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we // call __builtin_readcyclecounter. case PPC::BI__builtin_ppc_get_timebase: @@ -16980,37 +17411,34 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, } return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); } - // Rotate and insert under mask operation. - // __rldimi(rs, is, shift, mask) - // (rotl64(rs, shift) & mask) | (is & ~mask) - // __rlwimi(rs, is, shift, mask) - // (rotl(rs, shift) & mask) | (is & ~mask) case PPC::BI__builtin_ppc_rldimi: case PPC::BI__builtin_ppc_rlwimi: { Value *Op0 = EmitScalarExpr(E->getArg(0)); Value *Op1 = EmitScalarExpr(E->getArg(1)); Value *Op2 = EmitScalarExpr(E->getArg(2)); Value *Op3 = EmitScalarExpr(E->getArg(3)); - llvm::Type *Ty = Op0->getType(); - Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty); - if (BuiltinID == PPC::BI__builtin_ppc_rldimi) + // rldimi is 64-bit instruction, expand the intrinsic before isel to + // leverage peephole and avoid legalization efforts. + if (BuiltinID == PPC::BI__builtin_ppc_rldimi && + !getTarget().getTriple().isPPC64()) { + Function *F = CGM.getIntrinsic(Intrinsic::fshl, Op0->getType()); Op2 = Builder.CreateZExt(Op2, Int64Ty); - Value *Shift = Builder.CreateCall(F, {Op0, Op0, Op2}); - Value *X = Builder.CreateAnd(Shift, Op3); - Value *Y = Builder.CreateAnd(Op1, Builder.CreateNot(Op3)); - return Builder.CreateOr(X, Y); - } - // Rotate and insert under mask operation. - // __rlwnm(rs, shift, mask) - // rotl(rs, shift) & mask + Value *Shift = Builder.CreateCall(F, {Op0, Op0, Op2}); + return Builder.CreateOr(Builder.CreateAnd(Shift, Op3), + Builder.CreateAnd(Op1, Builder.CreateNot(Op3))); + } + return Builder.CreateCall( + CGM.getIntrinsic(BuiltinID == PPC::BI__builtin_ppc_rldimi + ? Intrinsic::ppc_rldimi + : Intrinsic::ppc_rlwimi), + {Op0, Op1, Op2, Op3}); + } case PPC::BI__builtin_ppc_rlwnm: { Value *Op0 = EmitScalarExpr(E->getArg(0)); Value *Op1 = EmitScalarExpr(E->getArg(1)); Value *Op2 = EmitScalarExpr(E->getArg(2)); - llvm::Type *Ty = Op0->getType(); - Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty); - Value *Shift = Builder.CreateCall(F, {Op0, Op0, Op1}); - return Builder.CreateAnd(Shift, Op2); + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_rlwnm), + {Op0, Op1, Op2}); } case PPC::BI__builtin_ppc_poppar4: case PPC::BI__builtin_ppc_poppar8: { @@ -17418,7 +17846,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, SmallVector<Value *, 4> Ops; for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) if (E->getArg(i)->getType()->isArrayType()) - Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); + Ops.push_back( + EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this)); else Ops.push_back(EmitScalarExpr(E->getArg(i))); // The first argument of these two builtins is a pointer used to store their @@ -17721,9 +18150,9 @@ Value *EmitAMDGPUImplicitArgPtr(CodeGenFunction &CGF) { // \p Index is 0, 1, and 2 for x, y, and z dimension, respectively. /// Emit code based on Code Object ABI version. /// COV_4 : Emit code to use dispatch ptr -/// COV_5 : Emit code to use implicitarg ptr +/// COV_5+ : Emit code to use implicitarg ptr /// COV_NONE : Emit code to load a global variable "__oclc_ABI_version" -/// and use its value for COV_4 or COV_5 approach. It is used for +/// and use its value for COV_4 or COV_5+ approach. It is used for /// compiling device libraries in an ABI-agnostic way. /// /// Note: "__oclc_ABI_version" is supposed to be emitted and intialized by @@ -17766,7 +18195,7 @@ Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) { Address(Result, CGF.Int16Ty, CharUnits::fromQuantity(2))); } else { Value *GEP = nullptr; - if (Cov == CodeObjectVersionKind::COV_5) { + if (Cov >= CodeObjectVersionKind::COV_5) { // Indexing the implicit kernarg segment. GEP = CGF.Builder.CreateConstGEP1_32( CGF.Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2); @@ -17837,9 +18266,35 @@ void CodeGenFunction::ProcessOrderScopeAMDGCN(Value *Order, Value *Scope, break; } + // Some of the atomic builtins take the scope as a string name. StringRef scp; - llvm::getConstantStringInfo(Scope, scp); - SSID = getLLVMContext().getOrInsertSyncScopeID(scp); + if (llvm::getConstantStringInfo(Scope, scp)) { + SSID = getLLVMContext().getOrInsertSyncScopeID(scp); + return; + } + + // Older builtins had an enum argument for the memory scope. + int scope = cast<llvm::ConstantInt>(Scope)->getZExtValue(); + switch (scope) { + case 0: // __MEMORY_SCOPE_SYSTEM + SSID = llvm::SyncScope::System; + break; + case 1: // __MEMORY_SCOPE_DEVICE + SSID = getLLVMContext().getOrInsertSyncScopeID("agent"); + break; + case 2: // __MEMORY_SCOPE_WRKGRP + SSID = getLLVMContext().getOrInsertSyncScopeID("workgroup"); + break; + case 3: // __MEMORY_SCOPE_WVFRNT + SSID = getLLVMContext().getOrInsertSyncScopeID("wavefront"); + break; + case 4: // __MEMORY_SCOPE_SINGLE + SSID = llvm::SyncScope::SingleThread; + break; + default: + SSID = llvm::SyncScope::System; + break; + } } llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments, @@ -17859,6 +18314,209 @@ llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments, return Arg; } +Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount) { + if (QT->hasFloatingRepresentation()) { + switch (elementCount) { + case 2: + return Intrinsic::dx_dot2; + case 3: + return Intrinsic::dx_dot3; + case 4: + return Intrinsic::dx_dot4; + } + } + if (QT->hasSignedIntegerRepresentation()) + return Intrinsic::dx_sdot; + + assert(QT->hasUnsignedIntegerRepresentation()); + return Intrinsic::dx_udot; +} + +Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + if (!getLangOpts().HLSL) + return nullptr; + + switch (BuiltinID) { + case Builtin::BI__builtin_hlsl_elementwise_all: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + return Builder.CreateIntrinsic( + /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()), + CGM.getHLSLRuntime().getAllIntrinsic(), ArrayRef<Value *>{Op0}, nullptr, + "hlsl.all"); + } + case Builtin::BI__builtin_hlsl_elementwise_any: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + return Builder.CreateIntrinsic( + /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()), + CGM.getHLSLRuntime().getAnyIntrinsic(), ArrayRef<Value *>{Op0}, nullptr, + "hlsl.any"); + } + case Builtin::BI__builtin_hlsl_elementwise_clamp: { + Value *OpX = EmitScalarExpr(E->getArg(0)); + Value *OpMin = EmitScalarExpr(E->getArg(1)); + Value *OpMax = EmitScalarExpr(E->getArg(2)); + + QualType Ty = E->getArg(0)->getType(); + bool IsUnsigned = false; + if (auto *VecTy = Ty->getAs<VectorType>()) + Ty = VecTy->getElementType(); + IsUnsigned = Ty->isUnsignedIntegerType(); + return Builder.CreateIntrinsic( + /*ReturnType=*/OpX->getType(), + IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp, + ArrayRef<Value *>{OpX, OpMin, OpMax}, nullptr, "dx.clamp"); + } + case Builtin::BI__builtin_hlsl_dot: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + Value *Op1 = EmitScalarExpr(E->getArg(1)); + llvm::Type *T0 = Op0->getType(); + llvm::Type *T1 = Op1->getType(); + if (!T0->isVectorTy() && !T1->isVectorTy()) { + if (T0->isFloatingPointTy()) + return Builder.CreateFMul(Op0, Op1, "dx.dot"); + + if (T0->isIntegerTy()) + return Builder.CreateMul(Op0, Op1, "dx.dot"); + + // Bools should have been promoted + llvm_unreachable( + "Scalar dot product is only supported on ints and floats."); + } + // A VectorSplat should have happened + assert(T0->isVectorTy() && T1->isVectorTy() && + "Dot product of vector and scalar is not supported."); + + // A vector sext or sitofp should have happened + assert(T0->getScalarType() == T1->getScalarType() && + "Dot product of vectors need the same element types."); + + auto *VecTy0 = E->getArg(0)->getType()->getAs<VectorType>(); + [[maybe_unused]] auto *VecTy1 = + E->getArg(1)->getType()->getAs<VectorType>(); + // A HLSLVectorTruncation should have happend + assert(VecTy0->getNumElements() == VecTy1->getNumElements() && + "Dot product requires vectors to be of the same size."); + + return Builder.CreateIntrinsic( + /*ReturnType=*/T0->getScalarType(), + getDotProductIntrinsic(E->getArg(0)->getType(), + VecTy0->getNumElements()), + ArrayRef<Value *>{Op0, Op1}, nullptr, "dx.dot"); + } break; + case Builtin::BI__builtin_hlsl_lerp: { + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + Value *S = EmitScalarExpr(E->getArg(2)); + if (!E->getArg(0)->getType()->hasFloatingRepresentation()) + llvm_unreachable("lerp operand must have a float representation"); + return Builder.CreateIntrinsic( + /*ReturnType=*/X->getType(), CGM.getHLSLRuntime().getLerpIntrinsic(), + ArrayRef<Value *>{X, Y, S}, nullptr, "hlsl.lerp"); + } + case Builtin::BI__builtin_hlsl_elementwise_frac: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + if (!E->getArg(0)->getType()->hasFloatingRepresentation()) + llvm_unreachable("frac operand must have a float representation"); + return Builder.CreateIntrinsic( + /*ReturnType=*/Op0->getType(), Intrinsic::dx_frac, + ArrayRef<Value *>{Op0}, nullptr, "dx.frac"); + } + case Builtin::BI__builtin_hlsl_elementwise_isinf: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + llvm::Type *Xty = Op0->getType(); + llvm::Type *retType = llvm::Type::getInt1Ty(this->getLLVMContext()); + if (Xty->isVectorTy()) { + auto *XVecTy = E->getArg(0)->getType()->getAs<VectorType>(); + retType = llvm::VectorType::get( + retType, ElementCount::getFixed(XVecTy->getNumElements())); + } + if (!E->getArg(0)->getType()->hasFloatingRepresentation()) + llvm_unreachable("isinf operand must have a float representation"); + return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf, + ArrayRef<Value *>{Op0}, nullptr, "dx.isinf"); + } + case Builtin::BI__builtin_hlsl_mad: { + Value *M = EmitScalarExpr(E->getArg(0)); + Value *A = EmitScalarExpr(E->getArg(1)); + Value *B = EmitScalarExpr(E->getArg(2)); + if (E->getArg(0)->getType()->hasFloatingRepresentation()) + return Builder.CreateIntrinsic( + /*ReturnType*/ M->getType(), Intrinsic::fmuladd, + ArrayRef<Value *>{M, A, B}, nullptr, "hlsl.fmad"); + + if (E->getArg(0)->getType()->hasSignedIntegerRepresentation()) { + if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil) + return Builder.CreateIntrinsic( + /*ReturnType*/ M->getType(), Intrinsic::dx_imad, + ArrayRef<Value *>{M, A, B}, nullptr, "dx.imad"); + + Value *Mul = Builder.CreateNSWMul(M, A); + return Builder.CreateNSWAdd(Mul, B); + } + assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation()); + if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil) + return Builder.CreateIntrinsic( + /*ReturnType=*/M->getType(), Intrinsic::dx_umad, + ArrayRef<Value *>{M, A, B}, nullptr, "dx.umad"); + + Value *Mul = Builder.CreateNUWMul(M, A); + return Builder.CreateNUWAdd(Mul, B); + } + case Builtin::BI__builtin_hlsl_elementwise_rcp: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + if (!E->getArg(0)->getType()->hasFloatingRepresentation()) + llvm_unreachable("rcp operand must have a float representation"); + llvm::Type *Ty = Op0->getType(); + llvm::Type *EltTy = Ty->getScalarType(); + Constant *One = Ty->isVectorTy() + ? ConstantVector::getSplat( + ElementCount::getFixed( + cast<FixedVectorType>(Ty)->getNumElements()), + ConstantFP::get(EltTy, 1.0)) + : ConstantFP::get(EltTy, 1.0); + return Builder.CreateFDiv(One, Op0, "hlsl.rcp"); + } + case Builtin::BI__builtin_hlsl_elementwise_rsqrt: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + if (!E->getArg(0)->getType()->hasFloatingRepresentation()) + llvm_unreachable("rsqrt operand must have a float representation"); + return Builder.CreateIntrinsic( + /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getRsqrtIntrinsic(), + ArrayRef<Value *>{Op0}, nullptr, "hlsl.rsqrt"); + } + case Builtin::BI__builtin_hlsl_wave_get_lane_index: { + return EmitRuntimeCall(CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, {}, false), "__hlsl_wave_get_lane_index", + {}, false, true)); + } + } + return nullptr; +} + +void CodeGenFunction::AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst, + const CallExpr *E) { + constexpr const char *Tag = "amdgpu-as"; + + LLVMContext &Ctx = Inst->getContext(); + SmallVector<MMRAMetadata::TagT, 3> MMRAs; + for (unsigned K = 2; K < E->getNumArgs(); ++K) { + llvm::Value *V = EmitScalarExpr(E->getArg(K)); + StringRef AS; + if (llvm::getConstantStringInfo(V, AS)) { + MMRAs.push_back({Tag, AS}); + // TODO: Delete the resulting unused constant? + continue; + } + CGM.Error(E->getExprLoc(), + "expected an address space name as a string literal"); + } + + llvm::sort(MMRAs); + MMRAs.erase(llvm::unique(MMRAs), MMRAs.end()); + Inst->setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, MMRAs)); +} + Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E) { llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent; @@ -17903,9 +18561,11 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, } case AMDGPU::BI__builtin_amdgcn_ds_swizzle: - return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_ds_swizzle); + return emitBuiltinWithOneOverloadedType<2>(*this, E, + Intrinsic::amdgcn_ds_swizzle); case AMDGPU::BI__builtin_amdgcn_mov_dpp8: - return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_mov_dpp8); + return emitBuiltinWithOneOverloadedType<2>(*this, E, + Intrinsic::amdgcn_mov_dpp8); case AMDGPU::BI__builtin_amdgcn_mov_dpp: case AMDGPU::BI__builtin_amdgcn_update_dpp: { llvm::SmallVector<llvm::Value *, 6> Args; @@ -17925,42 +18585,63 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, CGM.getIntrinsic(Intrinsic::amdgcn_update_dpp, Args[0]->getType()); return Builder.CreateCall(F, Args); } + case AMDGPU::BI__builtin_amdgcn_permlane16: + case AMDGPU::BI__builtin_amdgcn_permlanex16: + return emitBuiltinWithOneOverloadedType<6>( + *this, E, + BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16 + ? Intrinsic::amdgcn_permlane16 + : Intrinsic::amdgcn_permlanex16); + case AMDGPU::BI__builtin_amdgcn_permlane64: + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_permlane64); + case AMDGPU::BI__builtin_amdgcn_readlane: + return emitBuiltinWithOneOverloadedType<2>(*this, E, + Intrinsic::amdgcn_readlane); + case AMDGPU::BI__builtin_amdgcn_readfirstlane: + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_readfirstlane); case AMDGPU::BI__builtin_amdgcn_div_fixup: case AMDGPU::BI__builtin_amdgcn_div_fixupf: case AMDGPU::BI__builtin_amdgcn_div_fixuph: - return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_div_fixup); + return emitBuiltinWithOneOverloadedType<3>(*this, E, + Intrinsic::amdgcn_div_fixup); case AMDGPU::BI__builtin_amdgcn_trig_preop: case AMDGPU::BI__builtin_amdgcn_trig_preopf: return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop); case AMDGPU::BI__builtin_amdgcn_rcp: case AMDGPU::BI__builtin_amdgcn_rcpf: case AMDGPU::BI__builtin_amdgcn_rcph: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp); + return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_rcp); case AMDGPU::BI__builtin_amdgcn_sqrt: case AMDGPU::BI__builtin_amdgcn_sqrtf: case AMDGPU::BI__builtin_amdgcn_sqrth: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sqrt); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_sqrt); case AMDGPU::BI__builtin_amdgcn_rsq: case AMDGPU::BI__builtin_amdgcn_rsqf: case AMDGPU::BI__builtin_amdgcn_rsqh: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); + return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_rsq); case AMDGPU::BI__builtin_amdgcn_rsq_clamp: case AMDGPU::BI__builtin_amdgcn_rsq_clampf: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_rsq_clamp); case AMDGPU::BI__builtin_amdgcn_sinf: case AMDGPU::BI__builtin_amdgcn_sinh: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin); + return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_sin); case AMDGPU::BI__builtin_amdgcn_cosf: case AMDGPU::BI__builtin_amdgcn_cosh: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos); + return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_cos); case AMDGPU::BI__builtin_amdgcn_dispatch_ptr: return EmitAMDGPUDispatchPtr(*this, E); case AMDGPU::BI__builtin_amdgcn_logf: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log); + return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_log); case AMDGPU::BI__builtin_amdgcn_exp2f: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_exp2); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_exp2); case AMDGPU::BI__builtin_amdgcn_log_clampf: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_log_clamp); case AMDGPU::BI__builtin_amdgcn_ldexp: case AMDGPU::BI__builtin_amdgcn_ldexpf: { llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); @@ -17981,7 +18662,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, case AMDGPU::BI__builtin_amdgcn_frexp_mant: case AMDGPU::BI__builtin_amdgcn_frexp_mantf: case AMDGPU::BI__builtin_amdgcn_frexp_manth: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_frexp_mant); case AMDGPU::BI__builtin_amdgcn_frexp_exp: case AMDGPU::BI__builtin_amdgcn_frexp_expf: { Value *Src0 = EmitScalarExpr(E->getArg(0)); @@ -17998,13 +18680,17 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, case AMDGPU::BI__builtin_amdgcn_fract: case AMDGPU::BI__builtin_amdgcn_fractf: case AMDGPU::BI__builtin_amdgcn_fracth: - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::amdgcn_fract); case AMDGPU::BI__builtin_amdgcn_lerp: - return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_lerp); + return emitBuiltinWithOneOverloadedType<3>(*this, E, + Intrinsic::amdgcn_lerp); case AMDGPU::BI__builtin_amdgcn_ubfe: - return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_ubfe); + return emitBuiltinWithOneOverloadedType<3>(*this, E, + Intrinsic::amdgcn_ubfe); case AMDGPU::BI__builtin_amdgcn_sbfe: - return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_sbfe); + return emitBuiltinWithOneOverloadedType<3>(*this, E, + Intrinsic::amdgcn_sbfe); case AMDGPU::BI__builtin_amdgcn_ballot_w32: case AMDGPU::BI__builtin_amdgcn_ballot_w64: { llvm::Type *ResultType = ConvertType(E->getType()); @@ -18042,7 +18728,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class); case AMDGPU::BI__builtin_amdgcn_fmed3f: case AMDGPU::BI__builtin_amdgcn_fmed3h: - return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_fmed3); + return emitBuiltinWithOneOverloadedType<3>(*this, E, + Intrinsic::amdgcn_fmed3); case AMDGPU::BI__builtin_amdgcn_ds_append: case AMDGPU::BI__builtin_amdgcn_ds_consume: { Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ? @@ -18051,32 +18738,6 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, Function *F = CGM.getIntrinsic(Intrin, { Src0->getType() }); return Builder.CreateCall(F, { Src0, Builder.getFalse() }); } - case AMDGPU::BI__builtin_amdgcn_ds_faddf: - case AMDGPU::BI__builtin_amdgcn_ds_fminf: - case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: { - Intrinsic::ID Intrin; - switch (BuiltinID) { - case AMDGPU::BI__builtin_amdgcn_ds_faddf: - Intrin = Intrinsic::amdgcn_ds_fadd; - break; - case AMDGPU::BI__builtin_amdgcn_ds_fminf: - Intrin = Intrinsic::amdgcn_ds_fmin; - break; - case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: - Intrin = Intrinsic::amdgcn_ds_fmax; - break; - } - llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); - llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); - llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); - llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); - llvm::Value *Src4 = EmitScalarExpr(E->getArg(4)); - llvm::Function *F = CGM.getIntrinsic(Intrin, { Src1->getType() }); - llvm::FunctionType *FTy = F->getFunctionType(); - llvm::Type *PTy = FTy->getParamType(0); - Src0 = Builder.CreatePointerBitCastOrAddrSpaceCast(Src0, PTy); - return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 }); - } case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64: case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32: case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16: @@ -18149,74 +18810,46 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Function *F = CGM.getIntrinsic(IID, {Addr->getType()}); return Builder.CreateCall(F, {Addr, Val}); } - case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64: - case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: - case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: { + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: { + Intrinsic::ID IID; - llvm::Type *ArgTy; switch (BuiltinID) { - case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: - ArgTy = llvm::Type::getFloatTy(getLLVMContext()); - IID = Intrinsic::amdgcn_ds_fadd; - break; - case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64: - ArgTy = llvm::Type::getDoubleTy(getLLVMContext()); - IID = Intrinsic::amdgcn_ds_fadd; - break; - case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: - ArgTy = llvm::FixedVectorType::get( - llvm::Type::getHalfTy(getLLVMContext()), 2); - IID = Intrinsic::amdgcn_ds_fadd; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: + IID = Intrinsic::amdgcn_global_load_tr_b64; + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + IID = Intrinsic::amdgcn_global_load_tr_b128; break; } + llvm::Type *LoadTy = ConvertType(E->getType()); llvm::Value *Addr = EmitScalarExpr(E->getArg(0)); - llvm::Value *Val = EmitScalarExpr(E->getArg(1)); - llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue( - llvm::Type::getInt32Ty(getLLVMContext()), APInt(32, 0, true)); - llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue( - llvm::Type::getInt1Ty(getLLVMContext()), APInt(1, 0)); - llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy}); - return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1}); - } - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: { - - llvm::Type *ArgTy; - switch (BuiltinID) { - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: - ArgTy = llvm::Type::getInt32Ty(getLLVMContext()); - break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: - ArgTy = llvm::FixedVectorType::get( - llvm::Type::getInt32Ty(getLLVMContext()), 2); - break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: - ArgTy = llvm::FixedVectorType::get( - llvm::Type::getHalfTy(getLLVMContext()), 4); - break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: - ArgTy = llvm::FixedVectorType::get( - llvm::Type::getInt16Ty(getLLVMContext()), 4); - break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: - ArgTy = llvm::FixedVectorType::get( - llvm::Type::getHalfTy(getLLVMContext()), 8); - break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: - ArgTy = llvm::FixedVectorType::get( - llvm::Type::getInt16Ty(getLLVMContext()), 8); - break; - } - - llvm::Value *Addr = EmitScalarExpr(E->getArg(0)); - llvm::Function *F = - CGM.getIntrinsic(Intrinsic::amdgcn_global_load_tr, {ArgTy}); + llvm::Function *F = CGM.getIntrinsic(IID, {LoadTy}); return Builder.CreateCall(F, {Addr}); } + case AMDGPU::BI__builtin_amdgcn_get_fpenv: { + Function *F = CGM.getIntrinsic(Intrinsic::get_fpenv, + {llvm::Type::getInt64Ty(getLLVMContext())}); + return Builder.CreateCall(F); + } + case AMDGPU::BI__builtin_amdgcn_set_fpenv: { + Function *F = CGM.getIntrinsic(Intrinsic::set_fpenv, + {llvm::Type::getInt64Ty(getLLVMContext())}); + llvm::Value *Env = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall(F, {Env}); + } case AMDGPU::BI__builtin_amdgcn_read_exec: return EmitAMDGCNBallotForExec(*this, E, Int64Ty, Int64Ty, false); case AMDGPU::BI__builtin_amdgcn_read_exec_lo: @@ -18357,7 +18990,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12: case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12: AppendFalseForOpselArg = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32: case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64: ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB @@ -18366,7 +18999,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12: case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12: AppendFalseForOpselArg = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32: case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64: ArgsForMatchingMatrixTypes = {2, 0}; // CD, AB @@ -18519,7 +19152,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, // r600 intrinsics case AMDGPU::BI__builtin_r600_recipsqrt_ieee: case AMDGPU::BI__builtin_r600_recipsqrt_ieeef: - return emitUnaryBuiltin(*this, E, Intrinsic::r600_recipsqrt_ieee); + return emitBuiltinWithOneOverloadedType<1>(*this, E, + Intrinsic::r600_recipsqrt_ieee); case AMDGPU::BI__builtin_r600_read_tidig_x: return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024); case AMDGPU::BI__builtin_r600_read_tidig_y: @@ -18536,12 +19170,22 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, case AMDGPU::BI__builtin_amdgcn_fence: { ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), AO, SSID); - return Builder.CreateFence(AO, SSID); + FenceInst *Fence = Builder.CreateFence(AO, SSID); + if (E->getNumArgs() > 2) + AddAMDGPUFenceAddressSpaceMMRA(Fence, E); + return Fence; } case AMDGPU::BI__builtin_amdgcn_atomic_inc32: case AMDGPU::BI__builtin_amdgcn_atomic_inc64: case AMDGPU::BI__builtin_amdgcn_atomic_dec32: - case AMDGPU::BI__builtin_amdgcn_atomic_dec64: { + case AMDGPU::BI__builtin_amdgcn_atomic_dec64: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16: + case AMDGPU::BI__builtin_amdgcn_ds_faddf: + case AMDGPU::BI__builtin_amdgcn_ds_fminf: + case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: { llvm::AtomicRMWInst::BinOp BinOp; switch (BuiltinID) { case AMDGPU::BI__builtin_amdgcn_atomic_inc32: @@ -18552,23 +19196,62 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, case AMDGPU::BI__builtin_amdgcn_atomic_dec64: BinOp = llvm::AtomicRMWInst::UDecWrap; break; + case AMDGPU::BI__builtin_amdgcn_ds_faddf: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: + case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16: + BinOp = llvm::AtomicRMWInst::FAdd; + break; + case AMDGPU::BI__builtin_amdgcn_ds_fminf: + BinOp = llvm::AtomicRMWInst::FMin; + break; + case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: + BinOp = llvm::AtomicRMWInst::FMax; + break; } Address Ptr = CheckAtomicAlignment(*this, E); Value *Val = EmitScalarExpr(E->getArg(1)); + llvm::Type *OrigTy = Val->getType(); + QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); - ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)), - EmitScalarExpr(E->getArg(3)), AO, SSID); + bool Volatile; - QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); - bool Volatile = - PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); + if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_faddf || + BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fminf || + BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fmaxf) { + // __builtin_amdgcn_ds_faddf/fminf/fmaxf has an explicit volatile argument + Volatile = + cast<ConstantInt>(EmitScalarExpr(E->getArg(4)))->getZExtValue(); + } else { + // Infer volatile from the passed type. + Volatile = + PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); + } + + if (E->getNumArgs() >= 4) { + // Some of the builtins have explicit ordering and scope arguments. + ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)), + EmitScalarExpr(E->getArg(3)), AO, SSID); + } else { + // The ds_atomic_fadd_* builtins do not have syncscope/order arguments. + SSID = llvm::SyncScope::System; + AO = AtomicOrdering::SequentiallyConsistent; + + // The v2bf16 builtin uses i16 instead of a natural bfloat type. + if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16) { + llvm::Type *V2BF16Ty = FixedVectorType::get( + llvm::Type::getBFloatTy(Builder.getContext()), 2); + Val = Builder.CreateBitCast(Val, V2BF16Ty); + } + } llvm::AtomicRMWInst *RMW = Builder.CreateAtomicRMW(BinOp, Ptr, Val, AO, SSID); if (Volatile) RMW->setVolatile(true); - return RMW; + return Builder.CreateBitCast(RMW, OrigTy); } case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn: case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: { @@ -18579,6 +19262,50 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, CGM.getIntrinsic(Intrinsic::amdgcn_s_sendmsg_rtn, {ResultType}); return Builder.CreateCall(F, {Arg}); } + case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc: + return emitBuiltinWithOneOverloadedType<4>( + *this, E, Intrinsic::amdgcn_make_buffer_rsrc); + case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128: + return emitBuiltinWithOneOverloadedType<5>( + *this, E, Intrinsic::amdgcn_raw_ptr_buffer_store); + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96: + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: { + llvm::Type *RetTy = nullptr; + switch (BuiltinID) { + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8: + RetTy = Int8Ty; + break; + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16: + RetTy = Int16Ty; + break; + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32: + RetTy = Int32Ty; + break; + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64: + RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/2); + break; + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96: + RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/3); + break; + case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: + RetTy = llvm::FixedVectorType::get(Int32Ty, /*NumElements=*/4); + break; + } + Function *F = + CGM.getIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_load, RetTy); + return Builder.CreateCall( + F, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), + EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3))}); + } default: return nullptr; } @@ -19734,14 +20461,14 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, // Save returned values. assert(II.NumResults); if (II.NumResults == 1) { - Builder.CreateAlignedStore(Result, Dst.getPointer(), + Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this), CharUnits::fromQuantity(4)); } else { for (unsigned i = 0; i < II.NumResults; ++i) { Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), Dst.getElementType()), - Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), + Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); } @@ -19781,7 +20508,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < II.NumResults; ++i) { Value *V = Builder.CreateAlignedLoad( Src.getElementType(), - Builder.CreateGEP(Src.getElementType(), Src.getPointer(), + Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, ParamType)); @@ -19853,7 +20580,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsA; ++i) { Value *V = Builder.CreateAlignedLoad( SrcA.getElementType(), - Builder.CreateGEP(SrcA.getElementType(), SrcA.getPointer(), + Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, AType)); @@ -19863,7 +20590,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsB; ++i) { Value *V = Builder.CreateAlignedLoad( SrcB.getElementType(), - Builder.CreateGEP(SrcB.getElementType(), SrcB.getPointer(), + Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, BType)); @@ -19874,7 +20601,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsC; ++i) { Value *V = Builder.CreateAlignedLoad( SrcC.getElementType(), - Builder.CreateGEP(SrcC.getElementType(), SrcC.getPointer(), + Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, CType)); @@ -19884,7 +20611,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsD; ++i) Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType), - Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), + Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); return Result; @@ -20142,7 +20869,7 @@ struct BuiltinAlignArgs { BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) { QualType AstType = E->getArg(0)->getType(); if (AstType->isArrayType()) - Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF); else Src = CGF.EmitScalarExpr(E->getArg(0)); SrcType = Src->getType(); @@ -20318,6 +21045,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_min_f32: case WebAssembly::BI__builtin_wasm_min_f64: + case WebAssembly::BI__builtin_wasm_min_f16x8: case WebAssembly::BI__builtin_wasm_min_f32x4: case WebAssembly::BI__builtin_wasm_min_f64x2: { Value *LHS = EmitScalarExpr(E->getArg(0)); @@ -20328,6 +21056,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_max_f32: case WebAssembly::BI__builtin_wasm_max_f64: + case WebAssembly::BI__builtin_wasm_max_f16x8: case WebAssembly::BI__builtin_wasm_max_f32x4: case WebAssembly::BI__builtin_wasm_max_f64x2: { Value *LHS = EmitScalarExpr(E->getArg(0)); @@ -20336,6 +21065,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, CGM.getIntrinsic(Intrinsic::maximum, ConvertType(E->getType())); return Builder.CreateCall(Callee, {LHS, RHS}); } + case WebAssembly::BI__builtin_wasm_pmin_f16x8: case WebAssembly::BI__builtin_wasm_pmin_f32x4: case WebAssembly::BI__builtin_wasm_pmin_f64x2: { Value *LHS = EmitScalarExpr(E->getArg(0)); @@ -20344,6 +21074,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, CGM.getIntrinsic(Intrinsic::wasm_pmin, ConvertType(E->getType())); return Builder.CreateCall(Callee, {LHS, RHS}); } + case WebAssembly::BI__builtin_wasm_pmax_f16x8: case WebAssembly::BI__builtin_wasm_pmax_f32x4: case WebAssembly::BI__builtin_wasm_pmax_f64x2: { Value *LHS = EmitScalarExpr(E->getArg(0)); @@ -20642,6 +21373,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle); return Builder.CreateCall(Callee, Ops); } + case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8: + case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8: case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4: case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4: case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2: @@ -20651,10 +21384,12 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, Value *C = EmitScalarExpr(E->getArg(2)); unsigned IntNo; switch (BuiltinID) { + case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8: case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4: case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2: IntNo = Intrinsic::wasm_relaxed_madd; break; + case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8: case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4: case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: IntNo = Intrinsic::wasm_relaxed_nmadd; @@ -20758,9 +21493,31 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_bf16x8_add_f32); return Builder.CreateCall(Callee, {LHS, RHS, Acc}); } + case WebAssembly::BI__builtin_wasm_loadf16_f32: { + Value *Addr = EmitScalarExpr(E->getArg(0)); + Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_loadf16_f32); + return Builder.CreateCall(Callee, {Addr}); + } + case WebAssembly::BI__builtin_wasm_storef16_f32: { + Value *Val = EmitScalarExpr(E->getArg(0)); + Value *Addr = EmitScalarExpr(E->getArg(1)); + Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_storef16_f32); + return Builder.CreateCall(Callee, {Val, Addr}); + } + case WebAssembly::BI__builtin_wasm_splat_f16x8: { + Value *Val = EmitScalarExpr(E->getArg(0)); + Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_splat_f16x8); + return Builder.CreateCall(Callee, {Val}); + } + case WebAssembly::BI__builtin_wasm_extract_lane_f16x8: { + Value *Vector = EmitScalarExpr(E->getArg(0)); + Value *Index = EmitScalarExpr(E->getArg(1)); + Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_extract_lane_f16x8); + return Builder.CreateCall(Callee, {Vector, Index}); + } case WebAssembly::BI__builtin_wasm_table_get: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Function *Callee; if (E->getType().isWebAssemblyExternrefType()) @@ -20774,7 +21531,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_set: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Function *Callee; @@ -20789,13 +21546,13 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_size: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Value = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size); return Builder.CreateCall(Callee, Value); } case WebAssembly::BI__builtin_wasm_table_grow: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Val = EmitScalarExpr(E->getArg(1)); Value *NElems = EmitScalarExpr(E->getArg(2)); @@ -20812,7 +21569,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_fill: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Value *NElems = EmitScalarExpr(E->getArg(3)); @@ -20830,8 +21587,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_copy: { assert(E->getArg(0)->getType()->isArrayType()); - Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); - Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).getPointer(); + Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this); Value *DstIdx = EmitScalarExpr(E->getArg(2)); Value *SrcIdx = EmitScalarExpr(E->getArg(3)); Value *NElems = EmitScalarExpr(E->getArg(4)); @@ -20910,7 +21667,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) { // The base pointer is passed by address, so it needs to be loaded. Address A = EmitPointerWithAlignment(E->getArg(0)); - Address BP = Address(A.getPointer(), Int8PtrTy, A.getAlignment()); + Address BP = Address(A.emitRawPointer(*this), Int8PtrTy, A.getAlignment()); llvm::Value *Base = Builder.CreateLoad(BP); // The treatment of both loads and stores is the same: the arguments for // the builtin are the same as the arguments for the intrinsic. @@ -20951,8 +21708,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression // per call. Address DestAddr = EmitPointerWithAlignment(E->getArg(1)); - DestAddr = Address(DestAddr.getPointer(), Int8Ty, DestAddr.getAlignment()); - llvm::Value *DestAddress = DestAddr.getPointer(); + DestAddr = DestAddr.withElementType(Int8Ty); + llvm::Value *DestAddress = DestAddr.emitRawPointer(*this); // Operands are Base, Dest, Modifier. // The intrinsic format in LLVM IR is defined as @@ -21003,8 +21760,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } // These are identical to the builtins above, except they don't consume @@ -21022,8 +21779,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } @@ -21120,7 +21877,7 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, // Handle aggregate argument, namely RVV tuple types in segment load/store if (hasAggregateEvaluationKind(E->getArg(i)->getType())) { LValue L = EmitAggExprToLValue(E->getArg(i)); - llvm::Value *AggValue = Builder.CreateLoad(L.getAddress(*this)); + llvm::Value *AggValue = Builder.CreateLoad(L.getAddress()); Ops.push_back(AggValue); continue; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp index 5b43272bfa62..43dfbbb90dd5 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp @@ -71,8 +71,6 @@ private: bool RelocatableDeviceCode; /// Mangle context for device. std::unique_ptr<MangleContext> DeviceMC; - /// Some zeros used for GEPs. - llvm::Constant *Zeros[2]; llvm::FunctionCallee getSetupArgumentFn() const; llvm::FunctionCallee getLaunchFn() const; @@ -91,9 +89,7 @@ private: /// where the C code specifies const char*. llvm::Constant *makeConstantString(const std::string &Str, const std::string &Name = "") { - auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); - return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(), - ConstStr.getPointer(), Zeros); + return CGM.GetAddrOfConstantCString(Str, Name.c_str()).getPointer(); } /// Helper function which generates an initialized constant array from Str, @@ -117,7 +113,7 @@ private: } if (Alignment) GV->setAlignment(llvm::Align(Alignment)); - return llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); + return GV; } /// Helper function that generates an empty dummy function returning void. @@ -230,8 +226,6 @@ CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) IntTy = CGM.IntTy; SizeTy = CGM.SizeTy; VoidTy = CGM.VoidTy; - Zeros[0] = llvm::ConstantInt::get(SizeTy, 0); - Zeros[1] = Zeros[0]; PtrTy = CGM.UnqualPtrTy; } @@ -331,11 +325,11 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size()))); // Store pointers to the arguments in a locally allocated launch_args. for (unsigned i = 0; i < Args.size(); ++i) { - llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); + llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF); llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy); CGF.Builder.CreateDefaultAlignedStore( - VoidVarPtr, - CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i)); + VoidVarPtr, CGF.Builder.CreateConstGEP1_32( + PtrTy, KernelArgs.emitRawPointer(CGF), i)); } llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -361,7 +355,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, KernelLaunchAPI = KernelLaunchAPI + "_ptsz"; } auto LaunchKernelName = addPrefixToName(KernelLaunchAPI); - IdentifierInfo &cudaLaunchKernelII = + const IdentifierInfo &cudaLaunchKernelII = CGM.getContext().Idents.get(LaunchKernelName); FunctionDecl *cudaLaunchKernelFD = nullptr; for (auto *Result : DC->lookup(&cudaLaunchKernelII)) { @@ -393,9 +387,10 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, /*isVarArg=*/false), addUnderscoredPrefixToName("PopCallConfiguration")); - CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, - {GridDim.getPointer(), BlockDim.getPointer(), - ShmemSize.getPointer(), Stream.getPointer()}); + CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, {GridDim.emitRawPointer(CGF), + BlockDim.emitRawPointer(CGF), + ShmemSize.emitRawPointer(CGF), + Stream.emitRawPointer(CGF)}); // Emit the call to cudaLaunch llvm::Value *Kernel = @@ -405,7 +400,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, cudaLaunchKernelFD->getParamDecl(0)->getType()); LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty); LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty); - LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()), + LaunchKernelArgs.add(RValue::get(KernelArgs, CGF), cudaLaunchKernelFD->getParamDecl(3)->getType()); LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)), cudaLaunchKernelFD->getParamDecl(4)->getType()); @@ -423,6 +418,33 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, CGM.CreateRuntimeFunction(FTy, LaunchKernelName); CGF.EmitCall(FI, CGCallee::forDirect(cudaLaunchKernelFn), ReturnValueSlot(), LaunchKernelArgs); + + // To prevent CUDA device stub functions from being merged by ICF in MSVC + // environment, create an unique global variable for each kernel and write to + // the variable in the device stub. + if (CGM.getContext().getTargetInfo().getCXXABI().isMicrosoft() && + !CGF.getLangOpts().HIP) { + llvm::Function *KernelFunction = llvm::cast<llvm::Function>(Kernel); + std::string GlobalVarName = (KernelFunction->getName() + ".id").str(); + + llvm::GlobalVariable *HandleVar = + CGM.getModule().getNamedGlobal(GlobalVarName); + if (!HandleVar) { + HandleVar = new llvm::GlobalVariable( + CGM.getModule(), CGM.Int8Ty, + /*Constant=*/false, KernelFunction->getLinkage(), + llvm::ConstantInt::get(CGM.Int8Ty, 0), GlobalVarName); + HandleVar->setDSOLocal(KernelFunction->isDSOLocal()); + HandleVar->setVisibility(KernelFunction->getVisibility()); + if (KernelFunction->hasComdat()) + HandleVar->setComdat(CGM.getModule().getOrInsertComdat(GlobalVarName)); + } + + CGF.Builder.CreateAlignedStore(llvm::ConstantInt::get(CGM.Int8Ty, 1), + HandleVar, CharUnits::One(), + /*IsVolatile=*/true); + } + CGF.EmitBranch(EndBlock); CGF.EmitBlock(EndBlock); @@ -438,8 +460,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF, auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType()); Offset = Offset.alignTo(TInfo.Align); llvm::Value *Args[] = { - CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(), - PtrTy), + CGF.Builder.CreatePointerCast( + CGF.GetAddrOfLocalVar(A).emitRawPointer(CGF), PtrTy), llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), }; @@ -491,7 +513,8 @@ static void replaceManagedVar(llvm::GlobalVariable *Var, // variable with instructions. for (auto &&Op : WorkItem) { auto *CE = cast<llvm::ConstantExpr>(Op); - auto *NewInst = CE->getAsInstruction(I); + auto *NewInst = CE->getAsInstruction(); + NewInst->insertBefore(*I->getParent(), I->getIterator()); NewInst->replaceUsesOfWith(OldV, NewV); OldV = CE; NewV = NewInst; @@ -604,20 +627,10 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { uint64_t VarSize = CGM.getDataLayout().getTypeAllocSize(Var->getValueType()); if (Info.Flags.isManaged()) { - auto *ManagedVar = new llvm::GlobalVariable( - CGM.getModule(), Var->getType(), - /*isConstant=*/false, Var->getLinkage(), - /*Init=*/Var->isDeclaration() - ? nullptr - : llvm::ConstantPointerNull::get(Var->getType()), - /*Name=*/"", /*InsertBefore=*/nullptr, - llvm::GlobalVariable::NotThreadLocal); - ManagedVar->setDSOLocal(Var->isDSOLocal()); - ManagedVar->setVisibility(Var->getVisibility()); - ManagedVar->setExternallyInitialized(true); - ManagedVar->takeName(Var); - Var->setName(Twine(ManagedVar->getName() + ".managed")); - replaceManagedVar(Var, ManagedVar); + assert(Var->getName().ends_with(".managed") && + "HIP managed variables not transformed"); + auto *ManagedVar = CGM.getModule().getNamedGlobal( + Var->getName().drop_back(StringRef(".managed").size())); llvm::Value *Args[] = { &GpuBinaryHandlePtr, ManagedVar, @@ -760,10 +773,10 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // to contain the fat binary but will be populated somewhere else, // e.g. by lld through link script. FatBinStr = new llvm::GlobalVariable( - CGM.getModule(), CGM.Int8Ty, - /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, - "__hip_fatbin", nullptr, - llvm::GlobalVariable::NotThreadLocal); + CGM.getModule(), CGM.Int8Ty, + /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, + "__hip_fatbin_" + CGM.getContext().getCUIDHash(), nullptr, + llvm::GlobalVariable::NotThreadLocal); cast<llvm::GlobalVariable>(FatBinStr)->setSection(FatbinConstantName); } @@ -816,8 +829,8 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // thread safety of the loaded program. Therefore we can assume sequential // execution of constructor functions here. if (IsHIP) { - auto Linkage = CudaGpuBinary ? llvm::GlobalValue::InternalLinkage : - llvm::GlobalValue::LinkOnceAnyLinkage; + auto Linkage = CudaGpuBinary ? llvm::GlobalValue::InternalLinkage + : llvm::GlobalValue::ExternalLinkage; llvm::BasicBlock *IfBlock = llvm::BasicBlock::Create(Context, "if", ModuleCtorFunc); llvm::BasicBlock *ExitBlock = @@ -826,11 +839,11 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // of HIP ABI. GpuBinaryHandle = new llvm::GlobalVariable( TheModule, PtrTy, /*isConstant=*/false, Linkage, - /*Initializer=*/llvm::ConstantPointerNull::get(PtrTy), - "__hip_gpubin_handle"); - if (Linkage == llvm::GlobalValue::LinkOnceAnyLinkage) - GpuBinaryHandle->setComdat( - CGM.getModule().getOrInsertComdat(GpuBinaryHandle->getName())); + /*Initializer=*/ + CudaGpuBinary ? llvm::ConstantPointerNull::get(PtrTy) : nullptr, + CudaGpuBinary + ? "__hip_gpubin_handle" + : "__hip_gpubin_handle_" + CGM.getContext().getCUIDHash()); GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign()); // Prevent the weak symbol in different shared libraries being merged. if (Linkage != llvm::GlobalValue::InternalLinkage) @@ -1092,7 +1105,9 @@ void CGNVCUDARuntime::transformManagedVars() { : llvm::ConstantPointerNull::get(Var->getType()), /*Name=*/"", /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, - CGM.getContext().getTargetAddressSpace(LangAS::cuda_device)); + CGM.getContext().getTargetAddressSpace(CGM.getLangOpts().CUDAIsDevice + ? LangAS::cuda_device + : LangAS::Default)); ManagedVar->setDSOLocal(Var->isDSOLocal()); ManagedVar->setVisibility(Var->getVisibility()); ManagedVar->setExternallyInitialized(true); @@ -1101,7 +1116,7 @@ void CGNVCUDARuntime::transformManagedVars() { Var->setName(Twine(ManagedVar->getName()) + ".managed"); // Keep managed variables even if they are not used in device code since // they need to be allocated by the runtime. - if (!Var->isDeclaration()) { + if (CGM.getLangOpts().CUDAIsDevice && !Var->isDeclaration()) { assert(!ManagedVar->isDeclaration()); CGM.addCompilerUsedGlobal(Var); CGM.addCompilerUsedGlobal(ManagedVar); @@ -1159,9 +1174,8 @@ void CGNVCUDARuntime::createOffloadingEntries() { // Returns module constructor to be added. llvm::Function *CGNVCUDARuntime::finalizeModule() { + transformManagedVars(); if (CGM.getLangOpts().CUDAIsDevice) { - transformManagedVars(); - // Mark ODR-used device variables as compiler used to prevent it from being // eliminated by optimization. This is necessary for device variables // ODR-used by host functions. Sema correctly marks them as ODR-used no diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCUDARuntime.h b/contrib/llvm-project/clang/lib/CodeGen/CGCUDARuntime.h index c7af8f1cf0fe..8030d632cc3d 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCUDARuntime.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCUDARuntime.h @@ -54,10 +54,15 @@ public: }; private: + LLVM_PREFERRED_TYPE(DeviceVarKind) unsigned Kind : 2; + LLVM_PREFERRED_TYPE(bool) unsigned Extern : 1; + LLVM_PREFERRED_TYPE(bool) unsigned Constant : 1; // Constant variable. + LLVM_PREFERRED_TYPE(bool) unsigned Managed : 1; // Managed variable. + LLVM_PREFERRED_TYPE(bool) unsigned Normalized : 1; // Normalized texture. int SurfTexType; // Type of surface/texutre. diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCXX.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCXX.cpp index e95a735f92f7..23ebbee19bf7 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCXX.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCXX.cpp @@ -263,7 +263,16 @@ static CGCallee BuildAppleKextVirtualCall(CodeGenFunction &CGF, CGF.Builder.CreateConstInBoundsGEP1_64(Ty, VTable, VTableIndex, "vfnkxt"); llvm::Value *VFunc = CGF.Builder.CreateAlignedLoad( Ty, VFuncPtr, llvm::Align(CGF.PointerAlignInBytes)); - CGCallee Callee(GD, VFunc); + + CGPointerAuthInfo PointerAuth; + if (auto &Schema = + CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers) { + GlobalDecl OrigMD = + CGM.getItaniumVTableContext().findOriginalMethod(GD.getCanonicalDecl()); + PointerAuth = CGF.EmitPointerAuthInfo(Schema, VFuncPtr, OrigMD, QualType()); + } + + CGCallee Callee(GD, VFunc, PointerAuth); return Callee; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.cpp index a8bf57a277e9..7c6dfc3e59d8 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.cpp @@ -20,6 +20,12 @@ using namespace CodeGen; CGCXXABI::~CGCXXABI() { } +Address CGCXXABI::getThisAddress(CodeGenFunction &CGF) { + return CGF.makeNaturalAddressForPointer( + CGF.CXXABIThisValue, CGF.CXXABIThisDecl->getType()->getPointeeType(), + CGF.CXXABIThisAlignment); +} + void CGCXXABI::ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S) { DiagnosticsEngine &Diags = CGF.CGM.getDiags(); unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, @@ -44,8 +50,12 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer( llvm::Value *MemPtr, const MemberPointerType *MPT) { ErrorUnsupportedABI(CGF, "calls through member pointers"); - ThisPtrForCall = This.getPointer(); - const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>(); + const auto *RD = + cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl()); + ThisPtrForCall = + CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD)); + const FunctionProtoType *FPT = + MPT->getPointeeType()->getAs<FunctionProtoType>(); llvm::Constant *FnPtr = llvm::Constant::getNullValue( llvm::PointerType::getUnqual(CGM.getLLVMContext())); return CGCallee::forDirect(FnPtr, FPT); @@ -251,16 +261,15 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr, // If we don't need an array cookie, bail out early. if (!requiresArrayCookie(expr, eltTy)) { - allocPtr = ptr.getPointer(); + allocPtr = ptr.emitRawPointer(CGF); numElements = nullptr; cookieSize = CharUnits::Zero(); return; } cookieSize = getArrayCookieSizeImpl(eltTy); - Address allocAddr = - CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); - allocPtr = allocAddr.getPointer(); + Address allocAddr = CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); + allocPtr = allocAddr.emitRawPointer(CGF); numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.h b/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.h index ad1ad08d0856..7dcc53911199 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCXXABI.h @@ -57,12 +57,8 @@ protected: llvm::Value *getThisValue(CodeGenFunction &CGF) { return CGF.CXXABIThisValue; } - Address getThisAddress(CodeGenFunction &CGF) { - return Address( - CGF.CXXABIThisValue, - CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()), - CGF.CXXABIThisAlignment); - } + + Address getThisAddress(CodeGenFunction &CGF); /// Issue a diagnostic about unsupported features in the ABI. void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S); @@ -278,8 +274,7 @@ public: getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) = 0; virtual CatchTypeInfo getCatchAllTypeInfo(); - virtual bool shouldTypeidBeNullChecked(bool IsDeref, - QualType SrcRecordTy) = 0; + virtual bool shouldTypeidBeNullChecked(QualType SrcRecordTy) = 0; virtual void EmitBadTypeidCall(CodeGenFunction &CGF) = 0; virtual llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, Address ThisPtr, @@ -475,12 +470,6 @@ public: BaseSubobject Base, const CXXRecordDecl *NearestVBase) = 0; - /// Get the address point of the vtable for the given base subobject while - /// building a constexpr. - virtual llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) = 0; - /// Get the address of the vtable for the given record decl which should be /// used for the vptr at the given offset in RD. virtual llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, @@ -515,13 +504,15 @@ public: virtual void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD, bool ReturnAdjustment) = 0; - virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF, - Address This, - const ThisAdjustment &TA) = 0; + virtual llvm::Value * + performThisAdjustment(CodeGenFunction &CGF, Address This, + const CXXRecordDecl *UnadjustedClass, + const ThunkInfo &TI) = 0; - virtual llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, - Address Ret, - const ReturnAdjustment &RA) = 0; + virtual llvm::Value * + performReturnAdjustment(CodeGenFunction &CGF, Address Ret, + const CXXRecordDecl *UnadjustedClass, + const ReturnAdjustment &RA) = 0; virtual void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV, QualType ResultType); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp index 28c211aa631e..0416fa03d749 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp @@ -73,6 +73,10 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) { case CC_Swift: return llvm::CallingConv::Swift; case CC_SwiftAsync: return llvm::CallingConv::SwiftTail; case CC_M68kRTD: return llvm::CallingConv::M68k_RTD; + case CC_PreserveNone: return llvm::CallingConv::PreserveNone; + // clang-format off + case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall; + // clang-format on } } @@ -256,6 +260,12 @@ static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D, if (D->hasAttr<M68kRTDAttr>()) return CC_M68kRTD; + if (D->hasAttr<PreserveNoneAttr>()) + return CC_PreserveNone; + + if (D->hasAttr<RISCVVectorCCAttr>()) + return CC_RISCVVectorCall; + return CC_C; } @@ -304,7 +314,8 @@ CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) { if (MD->isImplicitObjectMemberFunction()) { // The abstract case is perfectly fine. - const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD); + const CXXRecordDecl *ThisType = + getCXXABI().getThisArgumentTypeForMethod(MD); return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD); } @@ -327,7 +338,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) { SmallVector<CanQualType, 16> argTypes; SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; - const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(GD); + const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD); argTypes.push_back(DeriveThisType(ThisType, MD)); bool PassParams = true; @@ -346,7 +357,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) { appendParameterTypes(*this, argTypes, paramInfos, FTP); CGCXXABI::AddedStructorArgCounts AddedArgs = - TheCXXABI.buildStructorSignature(GD, argTypes); + getCXXABI().buildStructorSignature(GD, argTypes); if (!paramInfos.empty()) { // Note: prefix implies after the first param. if (AddedArgs.Prefix) @@ -362,11 +373,10 @@ CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) { : RequiredArgs::All); FunctionType::ExtInfo extInfo = FTP->getExtInfo(); - CanQualType resultType = TheCXXABI.HasThisReturn(GD) - ? argTypes.front() - : TheCXXABI.hasMostDerivedReturn(GD) - ? CGM.getContext().VoidPtrTy - : Context.VoidTy; + CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front() + : getCXXABI().hasMostDerivedReturn(GD) + ? CGM.getContext().VoidPtrTy + : Context.VoidTy; return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::IsInstanceMethod, argTypes, extInfo, paramInfos, required); } @@ -427,11 +437,10 @@ CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args, : RequiredArgs::All; GlobalDecl GD(D, CtorKind); - CanQualType ResultType = TheCXXABI.HasThisReturn(GD) - ? ArgTypes.front() - : TheCXXABI.hasMostDerivedReturn(GD) - ? CGM.getContext().VoidPtrTy - : Context.VoidTy; + CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front() + : getCXXABI().hasMostDerivedReturn(GD) + ? CGM.getContext().VoidPtrTy + : Context.VoidTy; FunctionType::ExtInfo Info = FPT->getExtInfo(); llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos; @@ -796,7 +805,7 @@ const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo( } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) { swiftcall::computeABIInfo(CGM, *FI); } else { - getABIInfo().computeInfo(*FI); + CGM.getABIInfo().computeInfo(*FI); } // Loop over all of the computed argument and return value info. If any of @@ -929,8 +938,8 @@ struct NoExpansion : TypeExpansion { static std::unique_ptr<TypeExpansion> getTypeExpansion(QualType Ty, const ASTContext &Context) { if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) { - return std::make_unique<ConstantArrayExpansion>( - AT->getElementType(), AT->getSize().getZExtValue()); + return std::make_unique<ConstantArrayExpansion>(AT->getElementType(), + AT->getZExtSize()); } if (const RecordType *RT = Ty->getAs<RecordType>()) { SmallVector<const CXXBaseSpecifier *, 1> Bases; @@ -1027,15 +1036,9 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF, ConstantArrayExpansion *CAE, Address BaseAddr, llvm::function_ref<void(Address)> Fn) { - CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); - CharUnits EltAlign = - BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); - llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy); - for (int i = 0, n = CAE->NumElts; i < n; i++) { - llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32( - BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i); - Fn(Address(EltAddr, EltTy, EltAlign)); + Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i); + Fn(EltAddr); } } @@ -1047,12 +1050,12 @@ void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV, auto Exp = getTypeExpansion(Ty, getContext()); if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { forConstantArrayExpansion( - *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) { + *this, CAExp, LV.getAddress(), [&](Address EltAddr) { LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy); ExpandTypeFromArgs(CAExp->EltTy, LV, AI); }); } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { - Address This = LV.getAddress(*this); + Address This = LV.getAddress(); for (const CXXBaseSpecifier *BS : RExp->Bases) { // Perform a single step derived-to-base conversion. Address Base = @@ -1084,7 +1087,7 @@ void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV, // pointer type they use (see D118744). Once clang uses opaque pointers // all LLVM pointer types will be the same and we can remove this check. if (Arg->getType()->isPointerTy()) { - Address Addr = LV.getAddress(*this); + Address Addr = LV.getAddress(); Arg = Builder.CreateBitCast(Arg, Addr.getElementType()); } EmitStoreOfScalar(Arg, LV); @@ -1097,7 +1100,7 @@ void CodeGenFunction::ExpandTypeToArgs( SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) { auto Exp = getTypeExpansion(Ty, getContext()); if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { - Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this) + Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress() : Arg.getKnownRValue().getAggregateAddress(); forConstantArrayExpansion( *this, CAExp, Addr, [&](Address EltAddr) { @@ -1108,7 +1111,7 @@ void CodeGenFunction::ExpandTypeToArgs( IRCallArgPos); }); } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { - Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this) + Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress() : Arg.getKnownRValue().getAggregateAddress(); for (const CXXBaseSpecifier *BS : RExp->Bases) { // Perform a single step derived-to-base conversion. @@ -1150,9 +1153,10 @@ void CodeGenFunction::ExpandTypeToArgs( } /// Create a temporary allocation for the purposes of coercion. -static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, - CharUnits MinAlign, - const Twine &Name = "tmp") { +static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF, + llvm::Type *Ty, + CharUnits MinAlign, + const Twine &Name = "tmp") { // Don't use an alignment that's worse than what LLVM would prefer. auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty); CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); @@ -1297,111 +1301,84 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, // If coercing a fixed vector to a scalable vector for ABI compatibility, and // the types match, use the llvm.vector.insert intrinsic to perform the // conversion. - if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) { - if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) { - // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate + if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Ty)) { + if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(SrcTy)) { + // If we are casting a fixed i8 vector to a scalable i1 predicate // vector, use a vector insert and bitcast the result. - bool NeedsBitcast = false; - auto PredType = - llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16); - llvm::Type *OrigType = Ty; - if (ScalableDst == PredType && - FixedSrc->getElementType() == CGF.Builder.getInt8Ty()) { - ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2); - NeedsBitcast = true; + if (ScalableDstTy->getElementType()->isIntegerTy(1) && + ScalableDstTy->getElementCount().isKnownMultipleOf(8) && + FixedSrcTy->getElementType()->isIntegerTy(8)) { + ScalableDstTy = llvm::ScalableVectorType::get( + FixedSrcTy->getElementType(), + ScalableDstTy->getElementCount().getKnownMinValue() / 8); } - if (ScalableDst->getElementType() == FixedSrc->getElementType()) { + if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) { auto *Load = CGF.Builder.CreateLoad(Src); - auto *UndefVec = llvm::UndefValue::get(ScalableDst); + auto *UndefVec = llvm::UndefValue::get(ScalableDstTy); auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty); llvm::Value *Result = CGF.Builder.CreateInsertVector( - ScalableDst, UndefVec, Load, Zero, "cast.scalable"); - if (NeedsBitcast) - Result = CGF.Builder.CreateBitCast(Result, OrigType); + ScalableDstTy, UndefVec, Load, Zero, "cast.scalable"); + if (ScalableDstTy != Ty) + Result = CGF.Builder.CreateBitCast(Result, Ty); return Result; } } } // Otherwise do coercion through memory. This is stupid, but simple. - Address Tmp = + RawAddress Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName()); CGF.Builder.CreateMemCpy( - Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(), - Src.getAlignment().getAsAlign(), + Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), + Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue())); return CGF.Builder.CreateLoad(Tmp); } -// Function to store a first-class aggregate into memory. We prefer to -// store the elements rather than the aggregate to be more friendly to -// fast-isel. -// FIXME: Do we need to recurse here? -void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest, - bool DestIsVolatile) { - // Prefer scalar stores to first-class aggregate stores. - if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) { - for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - Address EltPtr = Builder.CreateStructGEP(Dest, i); - llvm::Value *Elt = Builder.CreateExtractValue(Val, i); - Builder.CreateStore(Elt, EltPtr, DestIsVolatile); - } - } else { - Builder.CreateStore(Val, Dest, DestIsVolatile); - } -} - -/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src, -/// where the source and destination may have different types. The -/// destination is known to be aligned to \arg DstAlign bytes. -/// -/// This safely handles the case when the src type is larger than the -/// destination type; the upper bits of the src will be lost. -static void CreateCoercedStore(llvm::Value *Src, - Address Dst, - bool DstIsVolatile, - CodeGenFunction &CGF) { - llvm::Type *SrcTy = Src->getType(); - llvm::Type *DstTy = Dst.getElementType(); - if (SrcTy == DstTy) { - CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); - return; - } - - llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); - - if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) { - Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, - SrcSize.getFixedValue(), CGF); - DstTy = Dst.getElementType(); - } - - llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy); - llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy); - if (SrcPtrTy && DstPtrTy && - SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) { - Src = CGF.Builder.CreateAddrSpaceCast(Src, DstTy); - CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); +void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, + llvm::TypeSize DstSize, + bool DstIsVolatile) { + if (!DstSize) return; - } - // If the source and destination are integer or pointer types, just do an - // extension or truncation to the desired type. - if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) && - (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) { - Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF); - CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); - return; + llvm::Type *SrcTy = Src->getType(); + llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy); + + // GEP into structs to try to make types match. + // FIXME: This isn't really that useful with opaque types, but it impacts a + // lot of regression tests. + if (SrcTy != Dst.getElementType()) { + if (llvm::StructType *DstSTy = + dyn_cast<llvm::StructType>(Dst.getElementType())) { + assert(!SrcSize.isScalable()); + Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, + SrcSize.getFixedValue(), *this); + } } - llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy); - - // If store is legal, just bitcast the src pointer. - if (isa<llvm::ScalableVectorType>(SrcTy) || - isa<llvm::ScalableVectorType>(DstTy) || - SrcSize.getFixedValue() <= DstSize.getFixedValue()) { - Dst = Dst.withElementType(SrcTy); - CGF.EmitAggregateStore(Src, Dst, DstIsVolatile); + if (SrcSize.isScalable() || SrcSize <= DstSize) { + if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() && + SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) { + // If the value is supposed to be a pointer, convert it before storing it. + Src = CoerceIntOrPtrToIntOrPtr(Src, Dst.getElementType(), *this); + Builder.CreateStore(Src, Dst, DstIsVolatile); + } else if (llvm::StructType *STy = + dyn_cast<llvm::StructType>(Src->getType())) { + // Prefer scalar stores to first-class aggregate stores. + Dst = Dst.withElementType(SrcTy); + for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { + Address EltPtr = Builder.CreateStructGEP(Dst, i); + llvm::Value *Elt = Builder.CreateExtractValue(Src, i); + Builder.CreateStore(Elt, EltPtr, DstIsVolatile); + } + } else { + Builder.CreateStore(Src, Dst.withElementType(SrcTy), DstIsVolatile); + } + } else if (SrcTy->isIntegerTy()) { + // If the source is a simple integer, coerce it directly. + llvm::Type *DstIntTy = Builder.getIntNTy(DstSize.getFixedValue() * 8); + Src = CoerceIntOrPtrToIntOrPtr(Src, DstIntTy, *this); + Builder.CreateStore(Src, Dst.withElementType(DstIntTy), DstIsVolatile); } else { // Otherwise do coercion through memory. This is stupid, but // simple. @@ -1412,12 +1389,13 @@ static void CreateCoercedStore(llvm::Value *Src, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); - CGF.Builder.CreateStore(Src, Tmp); - CGF.Builder.CreateMemCpy( - Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(), - Tmp.getAlignment().getAsAlign(), - llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue())); + RawAddress Tmp = + CreateTempAllocaForCoercion(*this, SrcTy, Dst.getAlignment()); + Builder.CreateStore(Src, Tmp); + Builder.CreateMemCpy(Dst.emitRawPointer(*this), + Dst.getAlignment().getAsAlign(), Tmp.getPointer(), + Tmp.getAlignment().getAsAlign(), + Builder.CreateTypeSize(IntPtrTy, DstSize)); } } @@ -1581,6 +1559,11 @@ bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) { return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet()); } +bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) { + const auto &RI = FI.getReturnInfo(); + return RI.getInReg(); +} + bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) { return ReturnTypeUsesSRet(FI) && getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs(); @@ -1774,14 +1757,14 @@ static void AddAttributesFromFunctionProtoType(ASTContext &Ctx, FuncAttrs.addAttribute("aarch64_pstate_sm_compatible"); // ZA - if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Out || - FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_InOut) - FuncAttrs.addAttribute("aarch64_pstate_za_shared"); - if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Preserves || - FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_In) { - FuncAttrs.addAttribute("aarch64_pstate_za_shared"); - FuncAttrs.addAttribute("aarch64_pstate_za_preserved"); - } + if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Preserves) + FuncAttrs.addAttribute("aarch64_preserves_za"); + if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_In) + FuncAttrs.addAttribute("aarch64_in_za"); + if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Out) + FuncAttrs.addAttribute("aarch64_out_za"); + if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_InOut) + FuncAttrs.addAttribute("aarch64_inout_za"); // ZT0 if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Preserves) @@ -1794,14 +1777,14 @@ static void AddAttributesFromFunctionProtoType(ASTContext &Ctx, FuncAttrs.addAttribute("aarch64_inout_zt0"); } -static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs, - const Decl *Callee) { +static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs, + const Decl *Callee) { if (!Callee) return; SmallVector<StringRef, 4> Attrs; - for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>()) + for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>()) AA->getAssumption().split(Attrs, ","); if (!Attrs.empty()) @@ -1908,6 +1891,7 @@ static void getTrivialDefaultFunctionAttributes( case CodeGenOptions::FramePointerKind::None: // This is the default behavior. break; + case CodeGenOptions::FramePointerKind::Reserved: case CodeGenOptions::FramePointerKind::NonLeaf: case CodeGenOptions::FramePointerKind::All: FuncAttrs.addAttribute("frame-pointer", @@ -2020,6 +2004,9 @@ static void getTrivialDefaultFunctionAttributes( std::tie(Var, Value) = Attr.split('='); FuncAttrs.addAttribute(Var, Value); } + + TargetInfo::BranchProtectionInfo BPI(LangOpts); + TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs); } /// Merges `target-features` from \TargetOpts and \F, and sets the result in @@ -2342,7 +2329,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name, // Attach assumption attributes to the declaration. If this is a call // site, attach assumptions from the caller to the call as well. - AddAttributesFromAssumes(FuncAttrs, TargetDecl); + AddAttributesFromOMPAssumes(FuncAttrs, TargetDecl); bool HasOptnone = false; // The NoBuiltinAttr attached to the target FunctionDecl. @@ -3016,17 +3003,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); - Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), - ArgI.getIndirectAlign(), KnownNonNull); + Address ParamAddr = makeNaturalAddressForPointer( + Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr, + nullptr, KnownNonNull); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we // need to do is realign the value, if requested. Also, if the address // may be aliased, copy it to ensure that the parameter variable is // mutable and has a unique adress, as C requires. - Address V = ParamAddr; if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) { - Address AlignedTemp = CreateMemTemp(Ty, "coerce"); + RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce"); // Copy from the incoming argument pointer to the temporary with the // appropriate alignment. @@ -3036,11 +3023,12 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, CharUnits Size = getContext().getTypeSizeInChars(Ty); Builder.CreateMemCpy( AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(), - ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(), + ParamAddr.emitRawPointer(*this), + ParamAddr.getAlignment().getAsAlign(), llvm::ConstantInt::get(IntPtrTy, Size.getQuantity())); - V = AlignedTemp; + ParamAddr = AlignedTemp; } - ArgVals.push_back(ParamValue::forIndirect(V)); + ArgVals.push_back(ParamValue::forIndirect(ParamAddr)); } else { // Load scalar value from indirect argument. llvm::Value *V = @@ -3084,7 +3072,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Align Alignment = CGM.getNaturalTypeAlignment(ETy).getAsAlign(); AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment)); - uint64_t ArrSize = ArrTy->getSize().getZExtValue(); + uint64_t ArrSize = ArrTy->getZExtSize(); if (!ETy->isIncompleteType() && ETy->isConstantSizeType() && ArrSize) { llvm::AttrBuilder Attrs(getLLVMContext()); @@ -3154,10 +3142,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, == ParameterABI::SwiftErrorResult) { QualType pointeeTy = Ty->getPointeeType(); assert(pointeeTy->isPointerType()); - Address temp = - CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); - Address arg(V, ConvertTypeForMem(pointeeTy), - getContext().getTypeAlignInChars(pointeeTy)); + RawAddress temp = + CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); + Address arg = makeNaturalAddressForPointer( + V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); llvm::Value *incomingErrorValue = Builder.CreateLoad(arg); Builder.CreateStore(incomingErrorValue, temp); V = temp.getPointer(); @@ -3195,13 +3183,14 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Value *Coerced = Fn->getArg(FirstIRArg); if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) { - // If we are casting a scalable 16 x i1 predicate vector to a fixed i8 + // If we are casting a scalable i1 predicate vector to a fixed i8 // vector, bitcast the source and use a vector extract. - auto PredType = - llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16); - if (VecTyFrom == PredType && + if (VecTyFrom->getElementType()->isIntegerTy(1) && + VecTyFrom->getElementCount().isKnownMultipleOf(8) && VecTyTo->getElementType() == Builder.getInt8Ty()) { - VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2); + VecTyFrom = llvm::ScalableVectorType::get( + VecTyTo->getElementType(), + VecTyFrom->getElementCount().getKnownMinValue() / 8); Coerced = Builder.CreateBitCast(Coerced, VecTyFrom); } if (VecTyFrom->getElementType() == VecTyTo->getElementType()) { @@ -3216,6 +3205,24 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, } } + llvm::StructType *STy = + dyn_cast<llvm::StructType>(ArgI.getCoerceToType()); + if (ArgI.isDirect() && !ArgI.getCanBeFlattened() && STy && + STy->getNumElements() > 1) { + [[maybe_unused]] llvm::TypeSize StructSize = + CGM.getDataLayout().getTypeAllocSize(STy); + [[maybe_unused]] llvm::TypeSize PtrElementSize = + CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(Ty)); + if (STy->containsHomogeneousScalableVectorTypes()) { + assert(StructSize == PtrElementSize && + "Only allow non-fractional movement of structure with" + "homogeneous scalable vector type"); + + ArgVals.push_back(ParamValue::forDirect(AI)); + break; + } + } + Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName()); @@ -3224,7 +3231,6 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, // Fast-isel and the optimizer generally like scalar values better than // FCAs, so we flatten them if this is safe to do for this argument. - llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType()); if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy && STy->getNumElements() > 1) { llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(STy); @@ -3277,7 +3283,12 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, assert(NumIRArgs == 1); auto AI = Fn->getArg(FirstIRArg); AI->setName(Arg->getName() + ".coerce"); - CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this); + CreateCoercedStore( + AI, Ptr, + llvm::TypeSize::getFixed( + getContext().getTypeSizeInChars(Ty).getQuantity() - + ArgI.getDirectOffset()), + /*DstIsVolatile=*/false); } // Match to what EmitParmDecl is expecting for this type. @@ -3476,7 +3487,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::LoadInst *load = dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts()); if (!load || load->isAtomic() || load->isVolatile() || - load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) + load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer()) return nullptr; // Okay! Burn it all down. This relies for correctness on the @@ -3513,12 +3524,15 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, /// Heuristically search for a dominating store to the return-value slot. static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { + llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer(); + // Check if a User is a store which pointerOperand is the ReturnValue. // We are looking for stores to the ReturnValue, not for stores of the // ReturnValue to some other location. - auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { + auto GetStoreIfValid = [&CGF, + ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * { auto *SI = dyn_cast<llvm::StoreInst>(U); - if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() || + if (!SI || SI->getPointerOperand() != ReturnValuePtr || SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()) return nullptr; // These aren't actually possible for non-coerced returns, and we @@ -3532,7 +3546,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { // for something immediately preceding the IP. Sometimes this can // happen with how we generate implicit-returns; it can also happen // with noreturn cleanups. - if (!CGF.ReturnValue.getPointer()->hasOneUse()) { + if (!ReturnValuePtr->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; @@ -3550,8 +3564,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { return nullptr; } - llvm::StoreInst *store = - GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); + llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back()); if (!store) return nullptr; // Now do a first-and-dirty dominance check: just walk up the @@ -3640,7 +3653,7 @@ static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset, for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) { const FieldDecl *F = *I; - if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context) || + if (F->isUnnamedBitField() || F->isZeroLengthBitField(Context) || F->getType()->isIncompleteArrayType()) continue; @@ -3829,7 +3842,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, LValue ArgVal = LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo); EmitStoreOfScalar( - Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true); + EmitLoadOfScalar(MakeAddrLValue(ReturnValue, RetTy), EndLoc), ArgVal, + /*isInit*/ true); break; } } @@ -4095,29 +4109,32 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, } static bool isProvablyNull(llvm::Value *addr) { - return isa<llvm::ConstantPointerNull>(addr); + return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(addr); +} + +static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) { + return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout()); } /// Emit the actual writing-back of a writeback. static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback) { const LValue &srcLV = writeback.Source; - Address srcAddr = srcLV.getAddress(CGF); - assert(!isProvablyNull(srcAddr.getPointer()) && + Address srcAddr = srcLV.getAddress(); + assert(!isProvablyNull(srcAddr.getBasePointer()) && "shouldn't have writeback for provably null argument"); llvm::BasicBlock *contBB = nullptr; // If the argument wasn't provably non-null, we need to null check // before doing the store. - bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), - CGF.CGM.getDataLayout()); + bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); + if (!provablyNonNull) { llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); contBB = CGF.createBasicBlock("icr.done"); - llvm::Value *isNull = - CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); + llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); CGF.EmitBlock(writebackBB); } @@ -4210,7 +4227,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType(); srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType); } - Address srcAddr = srcLV.getAddress(CGF); + Address srcAddr = srcLV.getAddress(); // The dest and src types don't necessarily match in LLVM terms // because of the crazy ObjC compatibility rules. @@ -4221,7 +4238,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, CGF.ConvertTypeForMem(CRE->getType()->getPointeeType()); // If the address is a constant null, just pass the appropriate null. - if (isProvablyNull(srcAddr.getPointer())) { + if (isProvablyNull(srcAddr.getBasePointer())) { args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), CRE->getType()); return; @@ -4250,17 +4267,16 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // If the address is *not* known to be non-null, we need to switch. llvm::Value *finalArgument; - bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), - CGF.CGM.getDataLayout()); + bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); + if (provablyNonNull) { - finalArgument = temp.getPointer(); + finalArgument = temp.emitRawPointer(CGF); } else { - llvm::Value *isNull = - CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); + llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); - finalArgument = CGF.Builder.CreateSelect(isNull, - llvm::ConstantPointerNull::get(destType), - temp.getPointer(), "icr.argument"); + finalArgument = CGF.Builder.CreateSelect( + isNull, llvm::ConstantPointerNull::get(destType), + temp.emitRawPointer(CGF), "icr.argument"); // If we need to copy, then the load has to be conditional, which // means we need control flow. @@ -4352,7 +4368,8 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo); bool CanCheckNullability = false; - if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) { + if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD && + !PVD->getType()->isRecordType()) { auto Nullability = PVD->getType()->getNullability(); CanCheckNullability = Nullability && *Nullability == NullabilityKind::NonNull && @@ -4384,6 +4401,16 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt); } +void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType, + SourceLocation ArgLoc, + AbstractCallee AC, unsigned ParmNum) { + if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) || + SanOpts.has(SanitizerKind::NullabilityArg))) + return; + + EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum); +} + // Check if the call is going to use the inalloca convention. This needs to // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged // later, so we can't check it directly. @@ -4606,7 +4633,7 @@ RValue CallArg::getRValue(CodeGenFunction &CGF) const { CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap, LV.isVolatile()); IsUsed = true; - return RValue::getAggregate(Copy.getAddress(CGF)); + return RValue::getAggregate(Copy.getAddress()); } void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const { @@ -4616,7 +4643,7 @@ void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const { else if (!HasLV && RV.isComplex()) CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true); else { - auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress(); + auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress(); LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty); // We assume that call args are never copied into subobjects. CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap, @@ -4655,11 +4682,11 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, AggValueSlot Slot = args.isUsingInAlloca() ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp"); - bool DestroyedInCallee = true, NeedsEHCleanup = true; + bool DestroyedInCallee = true, NeedsCleanup = true; if (const auto *RD = type->getAsCXXRecordDecl()) DestroyedInCallee = RD->hasNonTrivialDestructor(); else - NeedsEHCleanup = needsEHCleanup(type.isDestructedType()); + NeedsCleanup = type.isDestructedType(); if (DestroyedInCallee) Slot.setExternallyDestructed(); @@ -4668,21 +4695,23 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, RValue RV = Slot.asRValue(); args.add(RV, type); - if (DestroyedInCallee && NeedsEHCleanup) { + if (DestroyedInCallee && NeedsCleanup) { // Create a no-op GEP between the placeholder and the cleanup so we can // RAUW it successfully. It also serves as a marker of the first // instruction where the cleanup is active. - pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(), - type); + pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup, + Slot.getAddress(), type); // This unreachable is a temporary marker which will be removed later. - llvm::Instruction *IsActive = Builder.CreateUnreachable(); + llvm::Instruction *IsActive = + Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive); } return; } if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) && - cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) { + cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue && + !type->isArrayParameterType()) { LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr()); assert(L.isSimple()); args.addUncopiedAggregate(L, type); @@ -4724,12 +4753,22 @@ CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) { llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const llvm::Twine &name) { - return EmitNounwindRuntimeCall(callee, std::nullopt, name); + return EmitNounwindRuntimeCall(callee, ArrayRef<llvm::Value *>(), name); } /// Emits a call to the given nounwind runtime function. llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, + ArrayRef<Address> args, + const llvm::Twine &name) { + SmallVector<llvm::Value *, 3> values; + for (auto arg : args) + values.push_back(arg.emitRawPointer(*this)); + return EmitNounwindRuntimeCall(callee, values, name); +} + +llvm::CallInst * +CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args, const llvm::Twine &name) { llvm::CallInst *call = EmitRuntimeCall(callee, args, name); @@ -4775,6 +4814,9 @@ llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee, llvm::CallInst *call = Builder.CreateCall( callee, args, getBundlesForFunclet(callee.getCallee()), name); call->setCallingConv(getRuntimeCC()); + + if (CGM.shouldEmitConvergenceTokens() && call->isConvergent()) + return addControlledConvergenceToken(call); return call; } @@ -4971,7 +5013,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, ReturnValueSlot ReturnValue, const CallArgList &CallArgs, llvm::CallBase **callOrInvoke, bool IsMustTail, - SourceLocation Loc) { + SourceLocation Loc, + bool IsVirtualFunctionPointerThunk) { // FIXME: We no longer need the types from CallArgs; lift up and simplify. assert(Callee.isOrdinary() || Callee.isVirtual()); @@ -4995,18 +5038,19 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, (TargetDecl->hasAttr<TargetAttr>() || (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) checkTargetFeatures(Loc, FD); - - // Some architectures (such as x86-64) have the ABI changed based on - // attribute-target/features. Give them a chance to diagnose. - CGM.getTargetCodeGenInfo().checkFunctionCallABI( - CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs); } + // Some architectures (such as x86-64) have the ABI changed based on + // attribute-target/features. Give them a chance to diagnose. + CGM.getTargetCodeGenInfo().checkFunctionCallABI( + CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), + dyn_cast_or_null<FunctionDecl>(TargetDecl), CallArgs, RetTy); + // 1. Set up the arguments. // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - Address ArgMemory = Address::invalid(); + RawAddress ArgMemory = RawAddress::invalid(); if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { const llvm::DataLayout &DL = CGM.getDataLayout(); llvm::Instruction *IP = CallArgs.getStackBase(); @@ -5022,7 +5066,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, AI->setAlignment(Align.getAsAlign()); AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); - ArgMemory = Address(AI, ArgStruct, Align); + ArgMemory = RawAddress(AI, ArgStruct, Align); } ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); @@ -5031,11 +5075,15 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the call returns a temporary with struct return, create a temporary // alloca to hold the result, unless one is given to us. Address SRetPtr = Address::invalid(); - Address SRetAlloca = Address::invalid(); + RawAddress SRetAlloca = RawAddress::invalid(); llvm::Value *UnusedReturnSizePtr = nullptr; if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) { - if (!ReturnValue.isNull()) { - SRetPtr = ReturnValue.getValue(); + if (IsVirtualFunctionPointerThunk && RetAI.isIndirect()) { + SRetPtr = makeNaturalAddressForPointer(CurFn->arg_begin() + + IRFunctionArgs.getSRetArgNo(), + RetTy, CharUnits::fromQuantity(1)); + } else if (!ReturnValue.isNull()) { + SRetPtr = ReturnValue.getAddress(); } else { SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca); if (HaveInsertPoint() && ReturnValue.isUnused()) { @@ -5045,15 +5093,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } if (IRFunctionArgs.hasSRetArg()) { - IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); + IRCallArgs[IRFunctionArgs.getSRetArgNo()] = + getAsNaturalPointerTo(SRetPtr, RetTy); } else if (RetAI.isInAlloca()) { Address Addr = Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); - Builder.CreateStore(SRetPtr.getPointer(), Addr); + Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr); } } - Address swiftErrorTemp = Address::invalid(); + RawAddress swiftErrorTemp = RawAddress::invalid(); Address swiftErrorArg = Address::invalid(); // When passing arguments using temporary allocas, we need to add the @@ -5086,9 +5135,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 0); assert(getTarget().getTriple().getArch() == llvm::Triple::x86); if (I->isAggregate()) { - Address Addr = I->hasLValue() - ? I->getKnownLValue().getAddress(*this) - : I->getKnownRValue().getAggregateAddress(); + RawAddress Addr = I->hasLValue() + ? I->getKnownLValue().getAddress() + : I->getKnownRValue().getAggregateAddress(); llvm::Instruction *Placeholder = cast<llvm::Instruction>(Addr.getPointer()); @@ -5112,7 +5161,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } else if (ArgInfo.getInAllocaIndirect()) { // Make a temporary alloca and store the address of it into the argument // struct. - Address Addr = CreateMemTempWithoutCast( + RawAddress Addr = CreateMemTempWithoutCast( I->Ty, getContext().getTypeAlignInChars(I->Ty), "indirect-arg-temp"); I->copyInto(*this, Addr); @@ -5132,18 +5181,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); - if (!I->isAggregate()) { - // Make a temporary alloca to pass the argument. - Address Addr = CreateMemTempWithoutCast( - I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp"); - - llvm::Value *Val = Addr.getPointer(); - if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(Addr.getPointer()); - IRCallArgs[FirstIRArg] = Val; - - I->copyInto(*this, Addr); - } else { + if (I->isAggregate()) { // We want to avoid creating an unnecessary temporary+copy here; // however, we need one in three cases: // 1. If the argument is not byval, and we are required to copy the @@ -5153,9 +5191,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // 3. If the argument is byval, but RV is not located in default // or alloca address space. Address Addr = I->hasLValue() - ? I->getKnownLValue().getAddress(*this) + ? I->getKnownLValue().getAddress() : I->getKnownRValue().getAggregateAddress(); - llvm::Value *V = Addr.getPointer(); CharUnits Align = ArgInfo.getIndirectAlign(); const llvm::DataLayout *TD = &CGM.getDataLayout(); @@ -5166,8 +5203,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, bool NeedCopy = false; if (Addr.getAlignment() < Align && - llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) < - Align.getAsAlign()) { + llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this), + Align.getAsAlign(), + *TD) < Align.getAsAlign()) { NeedCopy = true; } else if (I->hasLValue()) { auto LV = I->getKnownLValue(); @@ -5196,29 +5234,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } - if (NeedCopy) { - // Create an aligned temporary, and copy to it. - Address AI = CreateMemTempWithoutCast( - I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); - llvm::Value *Val = AI.getPointer(); - if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(AI.getPointer()); - IRCallArgs[FirstIRArg] = Val; - - // Emit lifetime markers for the temporary alloca. - llvm::TypeSize ByvalTempElementSize = - CGM.getDataLayout().getTypeAllocSize(AI.getElementType()); - llvm::Value *LifetimeSize = - EmitLifetimeStart(ByvalTempElementSize, AI.getPointer()); - - // Add cleanup code to emit the end lifetime marker after the call. - if (LifetimeSize) // In case we disabled lifetime markers. - CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize); - - // Generate the copy. - I->copyInto(*this, AI); - } else { + if (!NeedCopy) { // Skip the extra memcpy call. + llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty); auto *T = llvm::PointerType::get( CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace()); @@ -5228,8 +5246,31 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (ArgHasMaybeUndefAttr) Val = Builder.CreateFreeze(Val); IRCallArgs[FirstIRArg] = Val; + break; } } + + // For non-aggregate args and aggregate args meeting conditions above + // we need to create an aligned temporary, and copy to it. + RawAddress AI = CreateMemTempWithoutCast( + I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); + llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty); + if (ArgHasMaybeUndefAttr) + Val = Builder.CreateFreeze(Val); + IRCallArgs[FirstIRArg] = Val; + + // Emit lifetime markers for the temporary alloca. + llvm::TypeSize ByvalTempElementSize = + CGM.getDataLayout().getTypeAllocSize(AI.getElementType()); + llvm::Value *LifetimeSize = + EmitLifetimeStart(ByvalTempElementSize, AI.getPointer()); + + // Add cleanup code to emit the end lifetime marker after the call. + if (LifetimeSize) // In case we disabled lifetime markers. + CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize); + + // Generate the copy. + I->copyInto(*this, AI); break; } @@ -5248,7 +5289,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, V = I->getKnownRValue().getScalarVal(); else V = Builder.CreateLoad( - I->hasLValue() ? I->getKnownLValue().getAddress(*this) + I->hasLValue() ? I->getKnownLValue().getAddress() : I->getKnownRValue().getAggregateAddress()); // Implement swifterror by copying into a new swifterror argument. @@ -5258,8 +5299,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(!swiftErrorTemp.isValid() && "multiple swifterror args"); QualType pointeeTy = I->Ty->getPointeeType(); - swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy), - getContext().getTypeAlignInChars(pointeeTy)); + swiftErrorArg = makeNaturalAddressForPointer( + V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); swiftErrorTemp = CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); @@ -5287,13 +5328,31 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, break; } + llvm::StructType *STy = + dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType()); + if (STy && ArgInfo.isDirect() && !ArgInfo.getCanBeFlattened()) { + llvm::Type *SrcTy = ConvertTypeForMem(I->Ty); + [[maybe_unused]] llvm::TypeSize SrcTypeSize = + CGM.getDataLayout().getTypeAllocSize(SrcTy); + [[maybe_unused]] llvm::TypeSize DstTypeSize = + CGM.getDataLayout().getTypeAllocSize(STy); + if (STy->containsHomogeneousScalableVectorTypes()) { + assert(SrcTypeSize == DstTypeSize && + "Only allow non-fractional movement of structure with " + "homogeneous scalable vector type"); + + IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal(); + break; + } + } + // FIXME: Avoid the conversion through memory if possible. Address Src = Address::invalid(); if (!I->isAggregate()) { Src = CreateMemTemp(I->Ty, "coerce"); I->copyInto(*this, Src); } else { - Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this) + Src = I->hasLValue() ? I->getKnownLValue().getAddress() : I->getKnownRValue().getAggregateAddress(); } @@ -5302,8 +5361,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // Fast-isel and the optimizer generally like scalar values better than // FCAs, so we flatten them if this is safe to do for this argument. - llvm::StructType *STy = - dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType()); if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) { llvm::Type *SrcTy = Src.getElementType(); llvm::TypeSize SrcTypeSize = @@ -5380,9 +5437,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *tempSize = nullptr; Address addr = Address::invalid(); - Address AllocaAddr = Address::invalid(); + RawAddress AllocaAddr = RawAddress::invalid(); if (I->isAggregate()) { - addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) + addr = I->hasLValue() ? I->getKnownLValue().getAddress() : I->getKnownRValue().getAggregateAddress(); } else { @@ -5524,6 +5581,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, /*AttrOnCallSite=*/true, /*IsThunk=*/false); + if (CallingConv == llvm::CallingConv::X86_VectorCall && + getTarget().getTriple().isWindowsArm64EC()) { + CGM.Error(Loc, "__vectorcall calling convention is not currently " + "supported"); + } + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) { if (FD->hasAttr<StrictFPAttr>()) // All calls within a strictfp function are marked strictfp @@ -5603,6 +5666,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, !isa_and_nonnull<FunctionDecl>(TargetDecl)) EmitKCFIOperandBundle(ConcreteCallee, BundleList); + // Add the pointer-authentication bundle. + EmitPointerAuthOperandBundle(ConcreteCallee.getPointerAuthInfo(), BundleList); + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) if (FD->hasAttr<StrictFPAttr>()) // All calls within a strictfp function are marked strictfp @@ -5650,6 +5716,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (!CI->getType()->isVoidTy()) CI->setName("call"); + if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent()) + CI = addControlledConvergenceToken(CI); + // Update largest vector width from the return type. LargestVectorWidth = std::max(LargestVectorWidth, getMaxVectorWidth(CI->getType())); @@ -5670,8 +5739,35 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) { if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>()) Call->setTailCallKind(llvm::CallInst::TCK_NoTail); - else if (IsMustTail) + else if (IsMustTail) { + if (getTarget().getTriple().isPPC()) { + if (getTarget().getTriple().isOSAIX()) + CGM.getDiags().Report(Loc, diag::err_aix_musttail_unsupported); + else if (!getTarget().hasFeature("pcrelative-memops")) { + if (getTarget().hasFeature("longcall")) + CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << 0; + else if (Call->isIndirectCall()) + CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << 1; + else if (isa_and_nonnull<FunctionDecl>(TargetDecl)) { + if (!cast<FunctionDecl>(TargetDecl)->isDefined()) + // The undefined callee may be a forward declaration. Without + // knowning all symbols in the module, we won't know the symbol is + // defined or not. Collect all these symbols for later diagnosing. + CGM.addUndefinedGlobalForTailCall( + {cast<FunctionDecl>(TargetDecl), Loc}); + else { + llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage( + GlobalDecl(cast<FunctionDecl>(TargetDecl))); + if (llvm::GlobalValue::isWeakForLinker(Linkage) || + llvm::GlobalValue::isDiscardableIfUnused(Linkage)) + CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) + << 2; + } + } + } + } Call->setTailCallKind(llvm::CallInst::TCK_MustTail); + } } // Add metadata for calls to MSAllocator functions @@ -5682,7 +5778,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // Add metadata if calling an __attribute__((error(""))) or warning fn. if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) { llvm::ConstantInt *Line = - llvm::ConstantInt::get(Int32Ty, Loc.getRawEncoding()); + llvm::ConstantInt::get(Int64Ty, Loc.getRawEncoding()); llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(Line); llvm::MDTuple *MDT = llvm::MDNode::get(getLLVMContext(), {MD}); CI->setMetadata("srcloc", MDT); @@ -5765,118 +5861,127 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, CallArgs.freeArgumentMemory(*this); // Extract the return value. - RValue Ret = [&] { - switch (RetAI.getKind()) { - case ABIArgInfo::CoerceAndExpand: { - auto coercionType = RetAI.getCoerceAndExpandType(); + RValue Ret; - Address addr = SRetPtr.withElementType(coercionType); - - assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType()); - bool requiresExtract = isa<llvm::StructType>(CI->getType()); - - unsigned unpaddedIndex = 0; - for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) { - llvm::Type *eltType = coercionType->getElementType(i); - if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue; - Address eltAddr = Builder.CreateStructGEP(addr, i); - llvm::Value *elt = CI; - if (requiresExtract) - elt = Builder.CreateExtractValue(elt, unpaddedIndex++); - else - assert(unpaddedIndex == 0); - Builder.CreateStore(elt, eltAddr); + // If the current function is a virtual function pointer thunk, avoid copying + // the return value of the musttail call to a temporary. + if (IsVirtualFunctionPointerThunk) { + Ret = RValue::get(CI); + } else { + Ret = [&] { + switch (RetAI.getKind()) { + case ABIArgInfo::CoerceAndExpand: { + auto coercionType = RetAI.getCoerceAndExpandType(); + + Address addr = SRetPtr.withElementType(coercionType); + + assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType()); + bool requiresExtract = isa<llvm::StructType>(CI->getType()); + + unsigned unpaddedIndex = 0; + for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) { + llvm::Type *eltType = coercionType->getElementType(i); + if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) + continue; + Address eltAddr = Builder.CreateStructGEP(addr, i); + llvm::Value *elt = CI; + if (requiresExtract) + elt = Builder.CreateExtractValue(elt, unpaddedIndex++); + else + assert(unpaddedIndex == 0); + Builder.CreateStore(elt, eltAddr); + } + [[fallthrough]]; } - [[fallthrough]]; - } - - case ABIArgInfo::InAlloca: - case ABIArgInfo::Indirect: { - RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation()); - if (UnusedReturnSizePtr) - PopCleanupBlock(); - return ret; - } - - case ABIArgInfo::Ignore: - // If we are ignoring an argument that had a result, make sure to - // construct the appropriate return value for our caller. - return GetUndefRValue(RetTy); - case ABIArgInfo::Extend: - case ABIArgInfo::Direct: { - llvm::Type *RetIRTy = ConvertType(RetTy); - if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) { - switch (getEvaluationKind(RetTy)) { - case TEK_Complex: { - llvm::Value *Real = Builder.CreateExtractValue(CI, 0); - llvm::Value *Imag = Builder.CreateExtractValue(CI, 1); - return RValue::getComplex(std::make_pair(Real, Imag)); - } - case TEK_Aggregate: { - Address DestPtr = ReturnValue.getValue(); - bool DestIsVolatile = ReturnValue.isVolatile(); + case ABIArgInfo::InAlloca: + case ABIArgInfo::Indirect: { + RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation()); + if (UnusedReturnSizePtr) + PopCleanupBlock(); + return ret; + } - if (!DestPtr.isValid()) { - DestPtr = CreateMemTemp(RetTy, "agg.tmp"); - DestIsVolatile = false; + case ABIArgInfo::Ignore: + // If we are ignoring an argument that had a result, make sure to + // construct the appropriate return value for our caller. + return GetUndefRValue(RetTy); + + case ABIArgInfo::Extend: + case ABIArgInfo::Direct: { + llvm::Type *RetIRTy = ConvertType(RetTy); + if (RetAI.getCoerceToType() == RetIRTy && + RetAI.getDirectOffset() == 0) { + switch (getEvaluationKind(RetTy)) { + case TEK_Complex: { + llvm::Value *Real = Builder.CreateExtractValue(CI, 0); + llvm::Value *Imag = Builder.CreateExtractValue(CI, 1); + return RValue::getComplex(std::make_pair(Real, Imag)); + } + case TEK_Aggregate: + break; + case TEK_Scalar: { + // If the argument doesn't match, perform a bitcast to coerce it. + // This can happen due to trivial type mismatches. + llvm::Value *V = CI; + if (V->getType() != RetIRTy) + V = Builder.CreateBitCast(V, RetIRTy); + return RValue::get(V); + } } - EmitAggregateStore(CI, DestPtr, DestIsVolatile); - return RValue::getAggregate(DestPtr); } - case TEK_Scalar: { - // If the argument doesn't match, perform a bitcast to coerce it. This - // can happen due to trivial type mismatches. + + // If coercing a fixed vector from a scalable vector for ABI + // compatibility, and the types match, use the llvm.vector.extract + // intrinsic to perform the conversion. + if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(RetIRTy)) { llvm::Value *V = CI; - if (V->getType() != RetIRTy) - V = Builder.CreateBitCast(V, RetIRTy); - return RValue::get(V); - } + if (auto *ScalableSrcTy = + dyn_cast<llvm::ScalableVectorType>(V->getType())) { + if (FixedDstTy->getElementType() == + ScalableSrcTy->getElementType()) { + llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty); + V = Builder.CreateExtractVector(FixedDstTy, V, Zero, + "cast.fixed"); + return RValue::get(V); + } + } } - llvm_unreachable("bad evaluation kind"); - } - // If coercing a fixed vector from a scalable vector for ABI - // compatibility, and the types match, use the llvm.vector.extract - // intrinsic to perform the conversion. - if (auto *FixedDst = dyn_cast<llvm::FixedVectorType>(RetIRTy)) { - llvm::Value *V = CI; - if (auto *ScalableSrc = dyn_cast<llvm::ScalableVectorType>(V->getType())) { - if (FixedDst->getElementType() == ScalableSrc->getElementType()) { - llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty); - V = Builder.CreateExtractVector(FixedDst, V, Zero, "cast.fixed"); - return RValue::get(V); - } + Address DestPtr = ReturnValue.getValue(); + bool DestIsVolatile = ReturnValue.isVolatile(); + uint64_t DestSize = + getContext().getTypeInfoDataSizeInChars(RetTy).Width.getQuantity(); + + if (!DestPtr.isValid()) { + DestPtr = CreateMemTemp(RetTy, "coerce"); + DestIsVolatile = false; + DestSize = getContext().getTypeSizeInChars(RetTy).getQuantity(); } - } - Address DestPtr = ReturnValue.getValue(); - bool DestIsVolatile = ReturnValue.isVolatile(); + // An empty record can overlap other data (if declared with + // no_unique_address); omit the store for such types - as there is no + // actual data to store. + if (!isEmptyRecord(getContext(), RetTy, true)) { + // If the value is offset in memory, apply the offset now. + Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); + CreateCoercedStore( + CI, StorePtr, + llvm::TypeSize::getFixed(DestSize - RetAI.getDirectOffset()), + DestIsVolatile); + } - if (!DestPtr.isValid()) { - DestPtr = CreateMemTemp(RetTy, "coerce"); - DestIsVolatile = false; + return convertTempToRValue(DestPtr, RetTy, SourceLocation()); } - // An empty record can overlap other data (if declared with - // no_unique_address); omit the store for such types - as there is no - // actual data to store. - if (!isEmptyRecord(getContext(), RetTy, true)) { - // If the value is offset in memory, apply the offset now. - Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); - CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this); + case ABIArgInfo::Expand: + case ABIArgInfo::IndirectAliased: + llvm_unreachable("Invalid ABI kind for return argument"); } - return convertTempToRValue(DestPtr, RetTy, SourceLocation()); - } - - case ABIArgInfo::Expand: - case ABIArgInfo::IndirectAliased: - llvm_unreachable("Invalid ABI kind for return argument"); - } - - llvm_unreachable("Unhandled ABIArgInfo::Kind"); - } (); + llvm_unreachable("Unhandled ABIArgInfo::Kind"); + }(); + } // Emit the assume_aligned check on the return value. if (Ret.isScalar() && TargetDecl) { @@ -5910,12 +6015,12 @@ CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const { /* VarArg handling */ -Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) { - VAListAddr = VE->isMicrosoftABI() - ? EmitMSVAListRef(VE->getSubExpr()) - : EmitVAListRef(VE->getSubExpr()); +RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr, + AggValueSlot Slot) { + VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(VE->getSubExpr()) + : EmitVAListRef(VE->getSubExpr()); QualType Ty = VE->getType(); if (VE->isMicrosoftABI()) - return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty); - return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty); + return CGM.getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty, Slot); + return CGM.getABIInfo().EmitVAArg(*this, VAListAddr, Ty, Slot); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCall.h b/contrib/llvm-project/clang/lib/CodeGen/CGCall.h index 1c0d15dc932a..412b44a8c753 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCall.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCall.h @@ -14,6 +14,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGCALL_H #define LLVM_CLANG_LIB_CODEGEN_CGCALL_H +#include "CGPointerAuthInfo.h" #include "CGValue.h" #include "EHScopeStack.h" #include "clang/AST/ASTFwd.h" @@ -69,6 +70,10 @@ class CGCallee { Last = Virtual }; + struct OrdinaryInfoStorage { + CGCalleeInfo AbstractInfo; + CGPointerAuthInfo PointerAuthInfo; + }; struct BuiltinInfoStorage { const FunctionDecl *Decl; unsigned ID; @@ -85,7 +90,7 @@ class CGCallee { SpecialKind KindOrFunctionPointer; union { - CGCalleeInfo AbstractInfo; + OrdinaryInfoStorage OrdinaryInfo; BuiltinInfoStorage BuiltinInfo; PseudoDestructorInfoStorage PseudoDestructorInfo; VirtualInfoStorage VirtualInfo; @@ -104,10 +109,13 @@ public: /// Construct a callee. Call this constructor directly when this /// isn't a direct call. - CGCallee(const CGCalleeInfo &abstractInfo, llvm::Value *functionPtr) + CGCallee(const CGCalleeInfo &abstractInfo, llvm::Value *functionPtr, + /* FIXME: make parameter pointerAuthInfo mandatory */ + const CGPointerAuthInfo &pointerAuthInfo = CGPointerAuthInfo()) : KindOrFunctionPointer( SpecialKind(reinterpret_cast<uintptr_t>(functionPtr))) { - AbstractInfo = abstractInfo; + OrdinaryInfo.AbstractInfo = abstractInfo; + OrdinaryInfo.PointerAuthInfo = pointerAuthInfo; assert(functionPtr && "configuring callee without function pointer"); assert(functionPtr->getType()->isPointerTy()); } @@ -173,7 +181,11 @@ public: if (isVirtual()) return VirtualInfo.MD; assert(isOrdinary()); - return AbstractInfo; + return OrdinaryInfo.AbstractInfo; + } + const CGPointerAuthInfo &getPointerAuthInfo() const { + assert(isOrdinary()); + return OrdinaryInfo.PointerAuthInfo; } llvm::Value *getFunctionPointer() const { assert(isOrdinary()); @@ -184,6 +196,10 @@ public: KindOrFunctionPointer = SpecialKind(reinterpret_cast<uintptr_t>(functionPtr)); } + void setPointerAuthInfo(CGPointerAuthInfo PointerAuth) { + assert(isOrdinary()); + OrdinaryInfo.PointerAuthInfo = PointerAuth; + } bool isVirtual() const { return KindOrFunctionPointer == SpecialKind::Virtual; @@ -357,8 +373,11 @@ class ReturnValueSlot { Address Addr = Address::invalid(); // Return value slot flags + LLVM_PREFERRED_TYPE(bool) unsigned IsVolatile : 1; + LLVM_PREFERRED_TYPE(bool) unsigned IsUnused : 1; + LLVM_PREFERRED_TYPE(bool) unsigned IsExternallyDestructed : 1; public: @@ -374,6 +393,7 @@ public: Address getValue() const { return Addr; } bool isUnused() const { return IsUnused; } bool isExternallyDestructed() const { return IsExternallyDestructed; } + Address getAddress() const { return Addr; } }; /// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp index 34319381901a..e5ba50de3462 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "ABIInfoImpl.h" #include "CGBlocks.h" #include "CGCXXABI.h" #include "CGDebugInfo.h" @@ -139,8 +140,9 @@ Address CodeGenFunction::LoadCXXThisAddress() { CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent()); } - llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType()); - return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull); + return makeNaturalAddressForPointer( + LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment, + false, nullptr, nullptr, KnownNonNull); } /// Emit the address of a field using a member data pointer. @@ -207,7 +209,7 @@ CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl, return nullptr; llvm::Type *PtrDiffTy = - Types.ConvertType(getContext().getPointerDiffType()); + getTypes().ConvertType(getContext().getPointerDiffType()); return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity()); } @@ -270,7 +272,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, } // Apply the base offset. - llvm::Value *ptr = addr.getPointer(); + llvm::Value *ptr = addr.emitRawPointer(CGF); ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr"); // If we have a virtual component, the alignment of the result will @@ -338,8 +340,8 @@ Address CodeGenFunction::GetAddressOfBaseClass( if (sanitizePerformTypeCheck()) { SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, !NullCheckValue); - EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), - DerivedTy, DerivedAlign, SkippedChecks); + EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy, + DerivedAlign, SkippedChecks); } return Value.withElementType(BaseValueTy); } @@ -354,7 +356,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer()); + llvm::Value *isNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); } @@ -363,14 +365,15 @@ Address CodeGenFunction::GetAddressOfBaseClass( SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, true); EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, - Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks); + Value.emitRawPointer(*this), DerivedTy, DerivedAlign, + SkippedChecks); } // Compute the virtual offset. llvm::Value *VirtualOffset = nullptr; if (VBase) { VirtualOffset = - CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); + CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); } // Apply both offsets. @@ -387,7 +390,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( EmitBlock(endBB); llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result"); - PHI->addIncoming(Value.getPointer(), notNullBB); + PHI->addIncoming(Value.emitRawPointer(*this), notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB); Value = Value.withPointer(PHI, NotKnownNonNull); } @@ -424,15 +427,19 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer()); + llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } // Apply the offset. - llvm::Value *Value = BaseAddr.getPointer(); - Value = Builder.CreateInBoundsGEP( - Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr"); + Address Addr = BaseAddr.withElementType(Int8Ty); + Addr = Builder.CreateInBoundsGEP( + Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty, + CGM.getClassPointerAlignment(Derived), "sub.ptr"); + + // Just cast. + Addr = Addr.withElementType(DerivedValueTy); // Produce a PHI if we had a null-check. if (NullCheckValue) { @@ -441,13 +448,15 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, Builder.CreateBr(CastEnd); EmitBlock(CastEnd); + llvm::Value *Value = Addr.emitRawPointer(*this); llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); - Value = PHI; + return Address(PHI, Addr.getElementType(), + CGM.getClassPointerAlignment(Derived)); } - return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived)); + return Addr; } llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, @@ -672,7 +681,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, // the constructor. QualType::DestructionKind dtorKind = FieldType.isDestructedType(); if (CGF.needsEHCleanup(dtorKind)) - CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType); + CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType); return; } } @@ -697,9 +706,9 @@ void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS, break; case TEK_Aggregate: { AggValueSlot Slot = AggValueSlot::forLValue( - LHS, *this, AggValueSlot::IsDestructed, - AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, - getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed, + LHS, AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, getOverlapForFieldInit(Field), + AggValueSlot::IsNotZeroed, // Checks are made by the code that calls constructor. AggValueSlot::IsSanitizerChecked); EmitAggExpr(Init, Slot); @@ -711,7 +720,7 @@ void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS, // later in the constructor. QualType::DestructionKind dtorKind = FieldType.isDestructedType(); if (needsEHCleanup(dtorKind)) - pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType); + pushEHDestroy(dtorKind, LHS.getAddress(), FieldType); } /// Checks whether the given constructor is a valid subject for the @@ -851,7 +860,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) { // Enter the function-try-block before the constructor prologue if // applicable. - bool IsTryBody = (Body && isa<CXXTryStmt>(Body)); + bool IsTryBody = isa_and_nonnull<CXXTryStmt>(Body); if (IsTryBody) EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true); @@ -925,7 +934,7 @@ namespace { } void addMemcpyableField(FieldDecl *F) { - if (F->isZeroSize(CGF.getContext())) + if (isEmptyFieldForLayout(CGF.getContext(), F)) return; if (!FirstField) addInitialField(F); @@ -975,8 +984,8 @@ namespace { LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField); emitMemcpyIR( - Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(CGF), - Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(CGF), + Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(), + Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(), MemcpySize); reset(); } @@ -1123,7 +1132,7 @@ namespace { continue; LValue FieldLHS = LHS; EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS); - CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType); + CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(), FieldType); } } @@ -1396,7 +1405,7 @@ FieldHasTrivialDestructorBody(ASTContext &Context, // The destructor for an implicit anonymous union member is never invoked. if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion()) - return false; + return true; return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl); } @@ -1467,7 +1476,7 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { // If the body is a function-try-block, enter the try before // anything else. - bool isTryBody = (Body && isa<CXXTryStmt>(Body)); + bool isTryBody = isa_and_nonnull<CXXTryStmt>(Body); if (isTryBody) EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true); EmitAsanPrologueOrEpilogue(false); @@ -1639,7 +1648,7 @@ namespace { LValue LV = CGF.EmitLValueForField(ThisLV, field); assert(LV.isSimple()); - CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer, + CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer, flags.isForNormalCleanup() && useEHCleanupForArray); } }; @@ -1719,7 +1728,7 @@ namespace { // Use the base class declaration location as inline DebugLocation. All // fields of the class are destroyed. DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass); - EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(), + EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF), BaseSize.getQuantity()); // Prevent the current stack frame from disappearing from the stack trace. @@ -1807,7 +1816,7 @@ namespace { const CXXDestructorDecl *DD) : Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {} void PushCleanupForField(const FieldDecl *Field) { - if (Field->isZeroSize(Context)) + if (isEmptyFieldForLayout(Context, Field)) return; unsigned FieldIndex = Field->getFieldIndex(); if (FieldHasTrivialDestructorBody(Context, Field)) { @@ -2022,7 +2031,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Find the end of the array. llvm::Type *elementType = arrayBase.getElementType(); - llvm::Value *arrayBegin = arrayBase.getPointer(); + llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this); llvm::Value *arrayEnd = Builder.CreateInBoundsGEP( elementType, arrayBegin, numElements, "arrayctor.end"); @@ -2118,14 +2127,15 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, Address This = ThisAVS.getAddress(); LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace(); LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace(); - llvm::Value *ThisPtr = This.getPointer(); + llvm::Value *ThisPtr = + getAsNaturalPointerTo(This, D->getThisType()->getPointeeType()); if (SlotAS != ThisAS) { unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS); llvm::Type *NewType = llvm::PointerType::get(getLLVMContext(), TargetThisAS); - ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(), - ThisAS, SlotAS, NewType); + ThisPtr = getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS, + SlotAS, NewType); } // Push the this ptr. @@ -2194,7 +2204,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, const CXXRecordDecl *ClassDecl = D->getParent(); if (!NewPointerIsChecked) - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(), + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, getContext().getRecordType(ClassDecl), CharUnits::Zero()); if (D->isTrivial() && D->isDefaultConstructor()) { @@ -2207,10 +2217,9 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // model that copy. if (isMemcpyEquivalentSpecialMember(D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); - QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); - Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy), - CGM.getNaturalTypeAlignment(SrcTy)); + Address Src = makeNaturalAddressForPointer( + Args[1].getRValue(*this).getScalarVal(), SrcTy); LValue SrcLVal = MakeAddrLValue(Src, SrcTy); QualType DestTy = getContext().getTypeDeclType(ClassDecl); LValue DestLVal = MakeAddrLValue(This, DestTy); @@ -2263,7 +2272,9 @@ void CodeGenFunction::EmitInheritedCXXConstructorCall( const CXXConstructorDecl *D, bool ForVirtualBase, Address This, bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) { CallArgList Args; - CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType()); + CallArg ThisArg(RValue::get(getAsNaturalPointerTo( + This, D->getThisType()->getPointeeType())), + D->getThisType()); // Forward the parameters. if (InheritedFromVBase && @@ -2388,12 +2399,14 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, CallArgList Args; // Push the this ptr. - Args.add(RValue::get(This.getPointer()), D->getThisType()); + Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())), + D->getThisType()); // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); - llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t); + llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType()); + llvm::Value *SrcVal = Builder.CreateBitCast(Val, t); Args.add(RValue::get(SrcVal), QT); // Skip over first argument (Src). @@ -2418,7 +2431,9 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor, // this Address This = LoadCXXThisAddress(); - DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType()); + DelegateArgs.add(RValue::get(getAsNaturalPointerTo( + This, (*I)->getType()->getPointeeType())), + (*I)->getType()); ++I; // FIXME: The location of the VTT parameter in the parameter list is @@ -2574,6 +2589,11 @@ void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) { // the same addr space. Note that this might not be LLVM address space 0. VTableField = VTableField.withElementType(PtrTy); + if (auto AuthenticationInfo = CGM.getVTablePointerAuthInfo( + this, Vptr.Base.getBase(), VTableField.emitRawPointer(*this))) + VTableAddressPoint = + EmitPointerAuthSign(*AuthenticationInfo, VTableAddressPoint); + llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField); TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(PtrTy); CGM.DecorateInstructionWithTBAA(Store, TBAAInfo); @@ -2667,12 +2687,35 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) { llvm::Value *CodeGenFunction::GetVTablePtr(Address This, llvm::Type *VTableTy, - const CXXRecordDecl *RD) { + const CXXRecordDecl *RD, + VTableAuthMode AuthMode) { Address VTablePtrSrc = This.withElementType(VTableTy); llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable"); TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy); CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo); + if (auto AuthenticationInfo = + CGM.getVTablePointerAuthInfo(this, RD, This.emitRawPointer(*this))) { + if (AuthMode != VTableAuthMode::UnsafeUbsanStrip) { + VTable = cast<llvm::Instruction>( + EmitPointerAuthAuth(*AuthenticationInfo, VTable)); + if (AuthMode == VTableAuthMode::MustTrap) { + // This is clearly suboptimal but until we have an ability + // to rely on the authentication intrinsic trapping and force + // an authentication to occur we don't really have a choice. + VTable = + cast<llvm::Instruction>(Builder.CreateBitCast(VTable, Int8PtrTy)); + Builder.CreateLoad(RawAddress(VTable, Int8Ty, CGM.getPointerAlign()), + /* IsVolatile */ true); + } + } else { + VTable = cast<llvm::Instruction>(EmitPointerAuthAuth( + CGPointerAuthInfo(0, PointerAuthenticationMode::Strip, false, false, + nullptr), + VTable)); + } + } + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && CGM.getCodeGenOpts().StrictVTablePointers) CGM.DecorateInstructionWithInvariantGroup(VTable, RD); @@ -2775,7 +2818,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived, if (MayBeNull) { llvm::Value *DerivedNotNull = - Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull"); + Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull"); llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); ContBlock = createBasicBlock("cast.cont"); @@ -2976,7 +3019,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda)); Address ThisPtr = GetAddrOfBlockDecl(variable); - CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType); // Add the rest of the parameters. for (auto *param : BD->parameters()) @@ -3004,7 +3047,7 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { QualType LambdaType = getContext().getRecordType(Lambda); QualType ThisType = getContext().getPointerType(LambdaType); Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); - CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType); EmitLambdaDelegatingInvokeBody(MD, CallArgs); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.cpp index f87caf050eea..4e210a9e3c95 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.cpp @@ -27,7 +27,7 @@ bool DominatingValue<RValue>::saved_type::needsSaving(RValue rv) { if (rv.isScalar()) return DominatingLLVMValue::needsSaving(rv.getScalarVal()); if (rv.isAggregate()) - return DominatingLLVMValue::needsSaving(rv.getAggregatePointer()); + return DominatingValue<Address>::needsSaving(rv.getAggregateAddress()); return true; } @@ -35,69 +35,40 @@ DominatingValue<RValue>::saved_type DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) { if (rv.isScalar()) { llvm::Value *V = rv.getScalarVal(); - - // These automatically dominate and don't need to be saved. - if (!DominatingLLVMValue::needsSaving(V)) - return saved_type(V, nullptr, ScalarLiteral); - - // Everything else needs an alloca. - Address addr = - CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue"); - CGF.Builder.CreateStore(V, addr); - return saved_type(addr.getPointer(), nullptr, ScalarAddress); + return saved_type(DominatingLLVMValue::save(CGF, V), + DominatingLLVMValue::needsSaving(V) ? ScalarAddress + : ScalarLiteral); } if (rv.isComplex()) { CodeGenFunction::ComplexPairTy V = rv.getComplexVal(); - llvm::Type *ComplexTy = - llvm::StructType::get(V.first->getType(), V.second->getType()); - Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex"); - CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); - CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); - return saved_type(addr.getPointer(), nullptr, ComplexAddress); + return saved_type(DominatingLLVMValue::save(CGF, V.first), + DominatingLLVMValue::save(CGF, V.second)); } assert(rv.isAggregate()); - Address V = rv.getAggregateAddress(); // TODO: volatile? - if (!DominatingLLVMValue::needsSaving(V.getPointer())) - return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral, - V.getAlignment().getQuantity()); - - Address addr = - CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue"); - CGF.Builder.CreateStore(V.getPointer(), addr); - return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress, - V.getAlignment().getQuantity()); + Address V = rv.getAggregateAddress(); + return saved_type(DominatingValue<Address>::save(CGF, V), + DominatingValue<Address>::needsSaving(V) + ? AggregateAddress + : AggregateLiteral); } /// Given a saved r-value produced by SaveRValue, perform the code /// necessary to restore it to usability at the current insertion /// point. RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) { - auto getSavingAddress = [&](llvm::Value *value) { - auto *AI = cast<llvm::AllocaInst>(value); - return Address(value, AI->getAllocatedType(), - CharUnits::fromQuantity(AI->getAlign().value())); - }; switch (K) { case ScalarLiteral: - return RValue::get(Value); case ScalarAddress: - return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value))); + return RValue::get(DominatingLLVMValue::restore(CGF, Vals.first)); case AggregateLiteral: + case AggregateAddress: return RValue::getAggregate( - Address(Value, ElementType, CharUnits::fromQuantity(Align))); - case AggregateAddress: { - auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value)); - return RValue::getAggregate( - Address(addr, ElementType, CharUnits::fromQuantity(Align))); - } + DominatingValue<Address>::restore(CGF, AggregateAddr)); case ComplexAddress: { - Address address = getSavingAddress(Value); - llvm::Value *real = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 0)); - llvm::Value *imag = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 1)); + llvm::Value *real = DominatingLLVMValue::restore(CGF, Vals.first); + llvm::Value *imag = DominatingLLVMValue::restore(CGF, Vals.second); return RValue::getComplex(real, imag); } } @@ -294,14 +265,14 @@ void EHScopeStack::popNullFixups() { BranchFixups.pop_back(); } -Address CodeGenFunction::createCleanupActiveFlag() { +RawAddress CodeGenFunction::createCleanupActiveFlag() { // Create a variable to decide whether the cleanup needs to be run. - Address active = CreateTempAllocaWithoutCast( + RawAddress active = CreateTempAllocaWithoutCast( Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond"); // Initialize it to false at a site that's guaranteed to be run // before each evaluation. - setBeforeOutermostConditional(Builder.getFalse(), active); + setBeforeOutermostConditional(Builder.getFalse(), active, *this); // Initialize it to true at the current location. Builder.CreateStore(Builder.getTrue(), active); @@ -309,7 +280,7 @@ Address CodeGenFunction::createCleanupActiveFlag() { return active; } -void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { +void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) { // Set that as the active flag in the cleanup. EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin()); assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?"); @@ -322,15 +293,17 @@ void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { void EHScopeStack::Cleanup::anchor() {} static void createStoreInstBefore(llvm::Value *value, Address addr, - llvm::Instruction *beforeInst) { - auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst); + llvm::Instruction *beforeInst, + CodeGenFunction &CGF) { + auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF), beforeInst); store->setAlignment(addr.getAlignment().getAsAlign()); } static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name, - llvm::Instruction *beforeInst) { - return new llvm::LoadInst(addr.getElementType(), addr.getPointer(), name, - false, addr.getAlignment().getAsAlign(), + llvm::Instruction *beforeInst, + CodeGenFunction &CGF) { + return new llvm::LoadInst(addr.getElementType(), addr.emitRawPointer(CGF), + name, false, addr.getAlignment().getAsAlign(), beforeInst); } @@ -357,8 +330,8 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF, // entry which we're currently popping. if (Fixup.OptimisticBranchBlock == nullptr) { createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex), - CGF.getNormalCleanupDestSlot(), - Fixup.InitialBranch); + CGF.getNormalCleanupDestSlot(), Fixup.InitialBranch, + CGF); Fixup.InitialBranch->setSuccessor(0, CleanupEntry); } @@ -385,7 +358,7 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF, if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) { assert(Br->isUnconditional()); auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(), - "cleanup.dest", Term); + "cleanup.dest", Term, CGF); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block); Br->eraseFromParent(); @@ -513,8 +486,8 @@ void CodeGenFunction::PopCleanupBlocks( I += Header.getSize(); if (Header.isConditional()) { - Address ActiveFlag = - reinterpret_cast<Address &>(LifetimeExtendedCleanupStack[I]); + RawAddress ActiveFlag = + reinterpret_cast<RawAddress &>(LifetimeExtendedCleanupStack[I]); initFullExprCleanupWithFlag(ActiveFlag); I += sizeof(ActiveFlag); } @@ -661,12 +634,19 @@ static void destroyOptimisticNormalEntry(CodeGenFunction &CGF, /// Pops a cleanup block. If the block includes a normal cleanup, the /// current insertion point is threaded through the cleanup, as are /// any branch fixups on the cleanup. -void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { +void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough, + bool ForDeactivation) { assert(!EHStack.empty() && "cleanup stack is empty!"); assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!"); EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin()); assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups()); + // If we are deactivating a normal cleanup, we need to pretend that the + // fallthrough is unreachable. We restore this IP before returning. + CGBuilderTy::InsertPoint NormalDeactivateOrigIP; + if (ForDeactivation && (Scope.isNormalCleanup() || !getLangOpts().EHAsynch)) { + NormalDeactivateOrigIP = Builder.saveAndClearIP(); + } // Remember activation information. bool IsActive = Scope.isActive(); Address NormalActiveFlag = @@ -694,7 +674,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // - whether there's a fallthrough llvm::BasicBlock *FallthroughSource = Builder.GetInsertBlock(); - bool HasFallthrough = (FallthroughSource != nullptr && IsActive); + bool HasFallthrough = + FallthroughSource != nullptr && (IsActive || HasExistingBranches); // Branch-through fall-throughs leave the insertion point set to the // end of the last cleanup, which points to the current scope. The @@ -719,7 +700,11 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // If we have a prebranched fallthrough into an inactive normal // cleanup, rewrite it so that it leads to the appropriate place. - if (Scope.isNormalCleanup() && HasPrebranchedFallthrough && !IsActive) { + if (Scope.isNormalCleanup() && HasPrebranchedFallthrough && + !RequiresNormalCleanup) { + // FIXME: Come up with a program which would need forwarding prebranched + // fallthrough and add tests. Otherwise delete this and assert against it. + assert(!IsActive); llvm::BasicBlock *prebranchDest; // If the prebranch is semantically branching through the next @@ -751,6 +736,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { EHStack.popCleanup(); // safe because there are no fixups assert(EHStack.getNumBranchFixups() == 0 || EHStack.hasNormalCleanups()); + if (NormalDeactivateOrigIP.isSet()) + Builder.restoreIP(NormalDeactivateOrigIP); return; } @@ -787,11 +774,19 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (!RequiresNormalCleanup) { // Mark CPP scope end for passed-by-value Arg temp // per Windows ABI which is "normally" Cleanup in callee - if (IsEHa && getInvokeDest() && Builder.GetInsertBlock()) { - if (Personality.isMSVCXXPersonality()) + if (IsEHa && getInvokeDest()) { + // If we are deactivating a normal cleanup then we don't have a + // fallthrough. Restore original IP to emit CPP scope ends in the correct + // block. + if (NormalDeactivateOrigIP.isSet()) + Builder.restoreIP(NormalDeactivateOrigIP); + if (Personality.isMSVCXXPersonality() && Builder.GetInsertBlock()) EmitSehCppScopeEnd(); + if (NormalDeactivateOrigIP.isSet()) + NormalDeactivateOrigIP = Builder.saveAndClearIP(); } destroyOptimisticNormalEntry(*this, Scope); + Scope.MarkEmitted(); EHStack.popCleanup(); } else { // If we have a fallthrough and no other need for the cleanup, @@ -808,6 +803,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { } destroyOptimisticNormalEntry(*this, Scope); + Scope.MarkEmitted(); EHStack.popCleanup(); EmitCleanup(*this, Fn, cleanupFlags, NormalActiveFlag); @@ -888,7 +884,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (NormalCleanupDestSlot->hasOneUse()) { NormalCleanupDestSlot->user_back()->eraseFromParent(); NormalCleanupDestSlot->eraseFromParent(); - NormalCleanupDest = Address::invalid(); + NormalCleanupDest = RawAddress::invalid(); } llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0); @@ -912,9 +908,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // pass the abnormal exit flag to Fn (SEH cleanup) cleanupFlags.setHasExitSwitch(); - llvm::LoadInst *Load = - createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest", - nullptr); + llvm::LoadInst *Load = createLoadInstBefore( + getNormalCleanupDestSlot(), "cleanup.dest", nullptr, *this); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Default, SwitchCapacity); @@ -944,6 +939,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { } // IV. Pop the cleanup and emit it. + Scope.MarkEmitted(); EHStack.popCleanup(); assert(EHStack.hasNormalCleanups() == HasEnclosingCleanups); @@ -961,8 +957,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (!Fixup.Destination) continue; if (!Fixup.OptimisticBranchBlock) { createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex), - getNormalCleanupDestSlot(), - Fixup.InitialBranch); + getNormalCleanupDestSlot(), Fixup.InitialBranch, + *this); Fixup.InitialBranch->setSuccessor(0, NormalEntry); } Fixup.OptimisticBranchBlock = NormalExit; @@ -1012,6 +1008,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { } } + if (NormalDeactivateOrigIP.isSet()) + Builder.restoreIP(NormalDeactivateOrigIP); assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0); // Emit the EH cleanup if required. @@ -1135,7 +1133,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) { // Store the index at the start. llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex()); - createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI); + createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI, *this); // Adjust BI to point to the first cleanup block. { @@ -1171,25 +1169,6 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) { Builder.ClearInsertionPoint(); } -static bool IsUsedAsNormalCleanup(EHScopeStack &EHStack, - EHScopeStack::stable_iterator C) { - // If we needed a normal block for any reason, that counts. - if (cast<EHCleanupScope>(*EHStack.find(C)).getNormalBlock()) - return true; - - // Check whether any enclosed cleanups were needed. - for (EHScopeStack::stable_iterator - I = EHStack.getInnermostNormalCleanup(); - I != C; ) { - assert(C.strictlyEncloses(I)); - EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I)); - if (S.getNormalBlock()) return true; - I = S.getEnclosingNormalCleanup(); - } - - return false; -} - static bool IsUsedAsEHCleanup(EHScopeStack &EHStack, EHScopeStack::stable_iterator cleanup) { // If we needed an EH block for any reason, that counts. @@ -1238,8 +1217,7 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, // Calculate whether the cleanup was used: // - as a normal cleanup - if (Scope.isNormalCleanup() && - (isActivatedInConditional || IsUsedAsNormalCleanup(CGF.EHStack, C))) { + if (Scope.isNormalCleanup()) { Scope.setTestFlagInNormalCleanup(); needFlag = true; } @@ -1252,13 +1230,16 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, } // If it hasn't yet been used as either, we're done. - if (!needFlag) return; + if (!needFlag) + return; Address var = Scope.getActiveFlag(); if (!var.isValid()) { + CodeGenFunction::AllocaTrackerRAII AllocaTracker(CGF); var = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), CharUnits::One(), "cleanup.isactive"); Scope.setActiveFlag(var); + Scope.AddAuxAllocas(AllocaTracker.Take()); assert(dominatingIP && "no existing variable and no dominating IP!"); @@ -1269,9 +1250,9 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, // If we're in a conditional block, ignore the dominating IP and // use the outermost conditional branch. if (CGF.isInConditionalBranch()) { - CGF.setBeforeOutermostConditional(value, var); + CGF.setBeforeOutermostConditional(value, var, CGF); } else { - createStoreInstBefore(value, var, dominatingIP); + createStoreInstBefore(value, var, dominatingIP, CGF); } } @@ -1301,17 +1282,8 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C, // to the current RunCleanupsScope. if (C == EHStack.stable_begin() && CurrentCleanupScopeDepth.strictlyEncloses(C)) { - // Per comment below, checking EHAsynch is not really necessary - // it's there to assure zero-impact w/o EHAsynch option - if (!Scope.isNormalCleanup() && getLangOpts().EHAsynch) { - PopCleanupBlock(); - } else { - // If it's a normal cleanup, we need to pretend that the - // fallthrough is unreachable. - CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP(); - PopCleanupBlock(); - Builder.restoreIP(SavedIP); - } + PopCleanupBlock(/*FallthroughIsBranchThrough=*/false, + /*ForDeactivation=*/true); return; } @@ -1321,7 +1293,7 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C, Scope.setActive(false); } -Address CodeGenFunction::getNormalCleanupDestSlot() { +RawAddress CodeGenFunction::getNormalCleanupDestSlot() { if (!NormalCleanupDest.isValid()) NormalCleanupDest = CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot"); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.h b/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.h index fcfbf41b0eaf..c73c97146abc 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCleanup.h @@ -16,8 +16,11 @@ #include "EHScopeStack.h" #include "Address.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Instruction.h" namespace llvm { class BasicBlock; @@ -40,6 +43,10 @@ struct CatchTypeInfo { /// A protected scope for zero-cost EH handling. class EHScope { +public: + enum Kind { Cleanup, Catch, Terminate, Filter }; + +private: llvm::BasicBlock *CachedLandingPad; llvm::BasicBlock *CachedEHDispatchBlock; @@ -47,6 +54,7 @@ class EHScope { class CommonBitFields { friend class EHScope; + LLVM_PREFERRED_TYPE(Kind) unsigned Kind : 3; }; enum { NumCommonBits = 3 }; @@ -64,21 +72,27 @@ protected: unsigned : NumCommonBits; /// Whether this cleanup needs to be run along normal edges. + LLVM_PREFERRED_TYPE(bool) unsigned IsNormalCleanup : 1; /// Whether this cleanup needs to be run along exception edges. + LLVM_PREFERRED_TYPE(bool) unsigned IsEHCleanup : 1; /// Whether this cleanup is currently active. + LLVM_PREFERRED_TYPE(bool) unsigned IsActive : 1; /// Whether this cleanup is a lifetime marker + LLVM_PREFERRED_TYPE(bool) unsigned IsLifetimeMarker : 1; /// Whether the normal cleanup should test the activation flag. + LLVM_PREFERRED_TYPE(bool) unsigned TestFlagInNormalCleanup : 1; /// Whether the EH cleanup should test the activation flag. + LLVM_PREFERRED_TYPE(bool) unsigned TestFlagInEHCleanup : 1; /// The amount of extra storage needed by the Cleanup. @@ -101,8 +115,6 @@ protected: }; public: - enum Kind { Cleanup, Catch, Terminate, Filter }; - EHScope(Kind kind, EHScopeStack::stable_iterator enclosingEHScope) : CachedLandingPad(nullptr), CachedEHDispatchBlock(nullptr), EnclosingEHScope(enclosingEHScope) { @@ -257,6 +269,51 @@ class alignas(8) EHCleanupScope : public EHScope { }; mutable struct ExtInfo *ExtInfo; + /// Erases auxillary allocas and their usages for an unused cleanup. + /// Cleanups should mark these allocas as 'used' if the cleanup is + /// emitted, otherwise these instructions would be erased. + struct AuxillaryAllocas { + SmallVector<llvm::Instruction *, 1> AuxAllocas; + bool used = false; + + // Records a potentially unused instruction to be erased later. + void Add(llvm::AllocaInst *Alloca) { AuxAllocas.push_back(Alloca); } + + // Mark all recorded instructions as used. These will not be erased later. + void MarkUsed() { + used = true; + AuxAllocas.clear(); + } + + ~AuxillaryAllocas() { + if (used) + return; + llvm::SetVector<llvm::Instruction *> Uses; + for (auto *Inst : llvm::reverse(AuxAllocas)) + CollectUses(Inst, Uses); + // Delete uses in the reverse order of insertion. + for (auto *I : llvm::reverse(Uses)) + I->eraseFromParent(); + } + + private: + void CollectUses(llvm::Instruction *I, + llvm::SetVector<llvm::Instruction *> &Uses) { + if (!I || !Uses.insert(I)) + return; + for (auto *User : I->users()) + CollectUses(cast<llvm::Instruction>(User), Uses); + } + }; + mutable struct AuxillaryAllocas *AuxAllocas; + + AuxillaryAllocas &getAuxillaryAllocas() { + if (!AuxAllocas) { + AuxAllocas = new struct AuxillaryAllocas(); + } + return *AuxAllocas; + } + /// The number of fixups required by enclosing scopes (not including /// this one). If this is the top cleanup scope, all the fixups /// from this index onwards belong to this scope. @@ -289,7 +346,7 @@ public: EHScopeStack::stable_iterator enclosingEH) : EHScope(EHScope::Cleanup, enclosingEH), EnclosingNormal(enclosingNormal), NormalBlock(nullptr), - ActiveFlag(Address::invalid()), ExtInfo(nullptr), + ActiveFlag(Address::invalid()), ExtInfo(nullptr), AuxAllocas(nullptr), FixupDepth(fixupDepth) { CleanupBits.IsNormalCleanup = isNormal; CleanupBits.IsEHCleanup = isEH; @@ -303,8 +360,15 @@ public: } void Destroy() { + if (AuxAllocas) + delete AuxAllocas; delete ExtInfo; } + void AddAuxAllocas(llvm::SmallVector<llvm::AllocaInst *> Allocas) { + for (auto *Alloca : Allocas) + getAuxillaryAllocas().Add(Alloca); + } + void MarkEmitted() { getAuxillaryAllocas().MarkUsed(); } // Objects of EHCleanupScope are not destructed. Use Destroy(). ~EHCleanupScope() = delete; @@ -324,7 +388,7 @@ public: Address getActiveFlag() const { return ActiveFlag; } - void setActiveFlag(Address Var) { + void setActiveFlag(RawAddress Var) { assert(Var.getAlignment().isOne()); ActiveFlag = Var; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCoroutine.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCoroutine.cpp index 888d30bfb3e1..a8a70186c2c5 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGCoroutine.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGCoroutine.cpp @@ -103,7 +103,7 @@ static void createCoroData(CodeGenFunction &CGF, return; } - CurCoro.Data = std::unique_ptr<CGCoroData>(new CGCoroData); + CurCoro.Data = std::make_unique<CGCoroData>(); CurCoro.Data->CoroId = CoroId; CurCoro.Data->CoroIdExpr = CoroIdExpr; } @@ -141,7 +141,7 @@ static bool FunctionCanThrow(const FunctionDecl *D) { Proto->canThrow() != CT_Cannot; } -static bool ResumeStmtCanThrow(const Stmt *S) { +static bool StmtCanThrow(const Stmt *S) { if (const auto *CE = dyn_cast<CallExpr>(S)) { const auto *Callee = CE->getDirectCallee(); if (!Callee) @@ -167,7 +167,7 @@ static bool ResumeStmtCanThrow(const Stmt *S) { } for (const auto *child : S->children()) - if (ResumeStmtCanThrow(child)) + if (StmtCanThrow(child)) return true; return false; @@ -178,18 +178,31 @@ static bool ResumeStmtCanThrow(const Stmt *S) { // auto && x = CommonExpr(); // if (!x.await_ready()) { // llvm_coro_save(); -// x.await_suspend(...); (*) -// llvm_coro_suspend(); (**) +// llvm_coro_await_suspend(&x, frame, wrapper) (*) (**) +// llvm_coro_suspend(); (***) // } // x.await_resume(); // // where the result of the entire expression is the result of x.await_resume() // -// (*) If x.await_suspend return type is bool, it allows to veto a suspend: +// (*) llvm_coro_await_suspend_{void, bool, handle} is lowered to +// wrapper(&x, frame) when it's certain not to interfere with +// coroutine transform. await_suspend expression is +// asynchronous to the coroutine body and not all analyses +// and transformations can handle it correctly at the moment. +// +// Wrapper function encapsulates x.await_suspend(...) call and looks like: +// +// auto __await_suspend_wrapper(auto& awaiter, void* frame) { +// std::coroutine_handle<> handle(frame); +// return awaiter.await_suspend(handle); +// } +// +// (**) If x.await_suspend return type is bool, it allows to veto a suspend: // if (x.await_suspend(...)) // llvm_coro_suspend(); // -// (**) llvm_coro_suspend() encodes three possible continuations as +// (***) llvm_coro_suspend() encodes three possible continuations as // a switch instruction: // // %where-to = call i8 @llvm.coro.suspend(...) @@ -212,9 +225,10 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co bool ignoreResult, bool forLValue) { auto *E = S.getCommonExpr(); - auto Binder = + auto CommonBinder = CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); - auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); + auto UnbindCommonOnExit = + llvm::make_scope_exit([&] { CommonBinder.unbind(CGF); }); auto Prefix = buildSuspendPrefixStr(Coro, Kind); BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready")); @@ -232,16 +246,74 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); + auto SuspendWrapper = CodeGenFunction(CGF.CGM).generateAwaitSuspendWrapper( + CGF.CurFn->getName(), Prefix, S); + CGF.CurCoro.InSuspendBlock = true; - auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); + + assert(CGF.CurCoro.Data && CGF.CurCoro.Data->CoroBegin && + "expected to be called in coroutine context"); + + SmallVector<llvm::Value *, 3> SuspendIntrinsicCallArgs; + SuspendIntrinsicCallArgs.push_back( + CGF.getOrCreateOpaqueLValueMapping(S.getOpaqueValue()).getPointer(CGF)); + + SuspendIntrinsicCallArgs.push_back(CGF.CurCoro.Data->CoroBegin); + SuspendIntrinsicCallArgs.push_back(SuspendWrapper); + + const auto SuspendReturnType = S.getSuspendReturnType(); + llvm::Intrinsic::ID AwaitSuspendIID; + + switch (SuspendReturnType) { + case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid: + AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_void; + break; + case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: + AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_bool; + break; + case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: + AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_handle; + break; + } + + llvm::Function *AwaitSuspendIntrinsic = CGF.CGM.getIntrinsic(AwaitSuspendIID); + + // SuspendHandle might throw since it also resumes the returned handle. + const bool AwaitSuspendCanThrow = + SuspendReturnType == + CoroutineSuspendExpr::SuspendReturnType::SuspendHandle || + StmtCanThrow(S.getSuspendExpr()); + + llvm::CallBase *SuspendRet = nullptr; + // FIXME: add call attributes? + if (AwaitSuspendCanThrow) + SuspendRet = + CGF.EmitCallOrInvoke(AwaitSuspendIntrinsic, SuspendIntrinsicCallArgs); + else + SuspendRet = CGF.EmitNounwindRuntimeCall(AwaitSuspendIntrinsic, + SuspendIntrinsicCallArgs); + + assert(SuspendRet); CGF.CurCoro.InSuspendBlock = false; - if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(1)) { + switch (SuspendReturnType) { + case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid: + assert(SuspendRet->getType()->isVoidTy()); + break; + case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: { + assert(SuspendRet->getType()->isIntegerTy()); + // Veto suspension if requested by bool returning await_suspend. BasicBlock *RealSuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend.bool")); CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); CGF.EmitBlock(RealSuspendBlock); + break; + } + case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: { + assert(SuspendRet->getType()->isVoidTy()); + break; + } } // Emit the suspend point. @@ -267,7 +339,7 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co // is marked as 'noexcept', we avoid generating this additional IR. CXXTryStmt *TryStmt = nullptr; if (Coro.ExceptionHandler && Kind == AwaitKind::Init && - ResumeStmtCanThrow(S.getResumeExpr())) { + StmtCanThrow(S.getResumeExpr())) { Coro.ResumeEHVar = CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh")); Builder.CreateFlagStore(true, Coro.ResumeEHVar); @@ -338,6 +410,67 @@ static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, } #endif +llvm::Function * +CodeGenFunction::generateAwaitSuspendWrapper(Twine const &CoroName, + Twine const &SuspendPointName, + CoroutineSuspendExpr const &S) { + std::string FuncName = + (CoroName + ".__await_suspend_wrapper__" + SuspendPointName).str(); + + ASTContext &C = getContext(); + + FunctionArgList args; + + ImplicitParamDecl AwaiterDecl(C, C.VoidPtrTy, ImplicitParamKind::Other); + ImplicitParamDecl FrameDecl(C, C.VoidPtrTy, ImplicitParamKind::Other); + QualType ReturnTy = S.getSuspendExpr()->getType(); + + args.push_back(&AwaiterDecl); + args.push_back(&FrameDecl); + + const CGFunctionInfo &FI = + CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args); + + llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI); + + llvm::Function *Fn = llvm::Function::Create( + LTy, llvm::GlobalValue::PrivateLinkage, FuncName, &CGM.getModule()); + + Fn->addParamAttr(0, llvm::Attribute::AttrKind::NonNull); + Fn->addParamAttr(0, llvm::Attribute::AttrKind::NoUndef); + + Fn->addParamAttr(1, llvm::Attribute::AttrKind::NoUndef); + + Fn->setMustProgress(); + Fn->addFnAttr(llvm::Attribute::AttrKind::AlwaysInline); + + StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args); + + // FIXME: add TBAA metadata to the loads + llvm::Value *AwaiterPtr = Builder.CreateLoad(GetAddrOfLocalVar(&AwaiterDecl)); + auto AwaiterLValue = + MakeNaturalAlignAddrLValue(AwaiterPtr, AwaiterDecl.getType()); + + CurAwaitSuspendWrapper.FramePtr = + Builder.CreateLoad(GetAddrOfLocalVar(&FrameDecl)); + + auto AwaiterBinder = CodeGenFunction::OpaqueValueMappingData::bind( + *this, S.getOpaqueValue(), AwaiterLValue); + + auto *SuspendRet = EmitScalarExpr(S.getSuspendExpr()); + + auto UnbindCommonOnExit = + llvm::make_scope_exit([&] { AwaiterBinder.unbind(*this); }); + if (SuspendRet != nullptr) { + Fn->addRetAttr(llvm::Attribute::AttrKind::NoUndef); + Builder.CreateStore(SuspendRet, ReturnValue); + } + + CurAwaitSuspendWrapper.FramePtr = nullptr; + FinishFunction(); + return Fn; +} + LValue CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) { assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && @@ -733,8 +866,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { EmitStmt(S.getPromiseDeclStmt()); Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); - auto *PromiseAddrVoidPtr = - new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); + auto *PromiseAddrVoidPtr = new llvm::BitCastInst( + PromiseAddr.emitRawPointer(*this), VoidPtrTy, "", CoroId); // Update CoroId to refer to the promise. We could not do it earlier because // promise local variable was not emitted yet. CoroId->setArgOperand(1, PromiseAddrVoidPtr); @@ -834,6 +967,11 @@ RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, if (CurCoro.Data && CurCoro.Data->CoroBegin) { return RValue::get(CurCoro.Data->CoroBegin); } + + if (CurAwaitSuspendWrapper.FramePtr) { + return RValue::get(CurAwaitSuspendWrapper.FramePtr); + } + CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_begin " "has been used earlier in this function"); auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getPtrTy()); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp index 0f3f684d61dc..3d8a715b692d 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp @@ -32,6 +32,7 @@ #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/Version.h" +#include "clang/CodeGen/ModuleBuilder.h" #include "clang/Frontend/FrontendOptions.h" #include "clang/Lex/HeaderSearchOptions.h" #include "clang/Lex/ModuleMap.h" @@ -58,7 +59,16 @@ using namespace clang::CodeGen; static uint32_t getTypeAlignIfRequired(const Type *Ty, const ASTContext &Ctx) { auto TI = Ctx.getTypeInfo(Ty); - return TI.isAlignRequired() ? TI.Align : 0; + if (TI.isAlignRequired()) + return TI.Align; + + // MaxFieldAlignmentAttr is the attribute added to types + // declared after #pragma pack(n). + if (auto *Decl = Ty->getAsRecordDecl()) + if (Decl->hasAttr<MaxFieldAlignmentAttr>()) + return TI.Align; + + return 0; } static uint32_t getTypeAlignIfRequired(QualType Ty, const ASTContext &Ctx) { @@ -856,7 +866,16 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { return SingletonId; \ } #include "clang/Basic/WebAssemblyReferenceTypes.def" - +#define AMDGPU_OPAQUE_PTR_TYPE(Name, MangledName, AS, Width, Align, Id, \ + SingletonId) \ + case BuiltinType::Id: { \ + if (!SingletonId) \ + SingletonId = \ + DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, \ + MangledName, TheCU, TheCU->getFile(), 0); \ + return SingletonId; \ + } +#include "clang/Basic/AMDGPUTypes.def" case BuiltinType::UChar: case BuiltinType::Char_U: Encoding = llvm::dwarf::DW_ATE_unsigned_char; @@ -1313,6 +1332,44 @@ llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty, return DBuilder.createPointerType(EltTy, Size); } +static llvm::SmallVector<TemplateArgument> +GetTemplateArgs(const TemplateDecl *TD, const TemplateSpecializationType *Ty) { + assert(Ty->isTypeAlias()); + // TemplateSpecializationType doesn't know if its template args are + // being substituted into a parameter pack. We can find out if that's + // the case now by inspecting the TypeAliasTemplateDecl template + // parameters. Insert Ty's template args into SpecArgs, bundling args + // passed to a parameter pack into a TemplateArgument::Pack. It also + // doesn't know the value of any defaulted args, so collect those now + // too. + SmallVector<TemplateArgument> SpecArgs; + ArrayRef SubstArgs = Ty->template_arguments(); + for (const NamedDecl *Param : TD->getTemplateParameters()->asArray()) { + // If Param is a parameter pack, pack the remaining arguments. + if (Param->isParameterPack()) { + SpecArgs.push_back(TemplateArgument(SubstArgs)); + break; + } + + // Skip defaulted args. + // FIXME: Ideally, we wouldn't do this. We can read the default values + // for each parameter. However, defaulted arguments which are dependent + // values or dependent types can't (easily?) be resolved here. + if (SubstArgs.empty()) { + // If SubstArgs is now empty (we're taking from it each iteration) and + // this template parameter isn't a pack, then that should mean we're + // using default values for the remaining template parameters (after + // which there may be an empty pack too which we will ignore). + break; + } + + // Take the next argument. + SpecArgs.push_back(SubstArgs.front()); + SubstArgs = SubstArgs.drop_front(); + } + return SpecArgs; +} + llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, llvm::DIFile *Unit) { assert(Ty->isTypeAlias()); @@ -1332,6 +1389,52 @@ llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, auto PP = getPrintingPolicy(); Ty->getTemplateName().print(OS, PP, TemplateName::Qualified::None); + SourceLocation Loc = AliasDecl->getLocation(); + + if (CGM.getCodeGenOpts().DebugTemplateAlias && + // FIXME: This is a workaround for the issue + // https://github.com/llvm/llvm-project/issues/89774 + // The TemplateSpecializationType doesn't contain any instantiation + // information; dependent template arguments can't be resolved. For now, + // fall back to DW_TAG_typedefs for template aliases that are + // instantiation dependent, e.g.: + // ``` + // template <int> + // using A = int; + // + // template<int I> + // struct S { + // using AA = A<I>; // Instantiation dependent. + // AA aa; + // }; + // + // S<0> s; + // ``` + // S::AA's underlying type A<I> is dependent on I so will be emitted as a + // DW_TAG_typedef. + !Ty->isInstantiationDependentType()) { + auto ArgVector = ::GetTemplateArgs(TD, Ty); + TemplateArgs Args = {TD->getTemplateParameters(), ArgVector}; + + // FIXME: Respect DebugTemplateNameKind::Mangled, e.g. by using GetName. + // Note we can't use GetName without additional work: TypeAliasTemplateDecl + // doesn't have instantiation information, so + // TypeAliasTemplateDecl::getNameForDiagnostic wouldn't have access to the + // template args. + std::string Name; + llvm::raw_string_ostream OS(Name); + TD->getNameForDiagnostic(OS, PP, /*Qualified=*/false); + if (CGM.getCodeGenOpts().getDebugSimpleTemplateNames() != + llvm::codegenoptions::DebugTemplateNamesKind::Simple || + !HasReconstitutableArgs(Args.Args)) + printTemplateArgumentList(OS, Args.Args, PP); + + llvm::DIDerivedType *AliasTy = DBuilder.createTemplateAlias( + Src, Name, getOrCreateFile(Loc), getLineNumber(Loc), + getDeclContextDescriptor(AliasDecl), CollectTemplateParams(Args, Unit)); + return AliasTy; + } + // Disable PrintCanonicalTypes here because we want // the DW_AT_name to benefit from the TypePrinter's ability // to skip defaulted template arguments. @@ -1343,8 +1446,6 @@ llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, PP.PrintCanonicalTypes = false; printTemplateArgumentList(OS, Ty->template_arguments(), PP, TD->getTemplateParameters()); - - SourceLocation Loc = AliasDecl->getLocation(); return DBuilder.createTypedef(Src, OS.str(), getOrCreateFile(Loc), getLineNumber(Loc), getDeclContextDescriptor(AliasDecl)); @@ -1440,8 +1541,7 @@ static unsigned getDwarfCC(CallingConv CC) { case CC_Swift: return llvm::dwarf::DW_CC_LLVM_Swift; case CC_SwiftAsync: - // [FIXME: swiftasynccc] Update to SwiftAsync once LLVM support lands. - return llvm::dwarf::DW_CC_LLVM_Swift; + return llvm::dwarf::DW_CC_LLVM_SwiftTail; case CC_PreserveMost: return llvm::dwarf::DW_CC_LLVM_PreserveMost; case CC_PreserveAll: @@ -1450,6 +1550,10 @@ static unsigned getDwarfCC(CallingConv CC) { return llvm::dwarf::DW_CC_LLVM_X86RegCall; case CC_M68kRTD: return llvm::dwarf::DW_CC_LLVM_M68kRTD; + case CC_PreserveNone: + return llvm::dwarf::DW_CC_LLVM_PreserveNone; + case CC_RISCVVectorCall: + return llvm::dwarf::DW_CC_LLVM_RISCVVectorCall; } return 0; } @@ -1628,6 +1732,28 @@ llvm::DIType *CGDebugInfo::createFieldType( offsetInBits, flags, debugType, Annotations); } +llvm::DISubprogram * +CGDebugInfo::createInlinedTrapSubprogram(StringRef FuncName, + llvm::DIFile *FileScope) { + // We are caching the subprogram because we don't want to duplicate + // subprograms with the same message. Note that `SPFlagDefinition` prevents + // subprograms from being uniqued. + llvm::DISubprogram *&SP = InlinedTrapFuncMap[FuncName]; + + if (!SP) { + llvm::DISubroutineType *DIFnTy = DBuilder.createSubroutineType(nullptr); + SP = DBuilder.createFunction( + /*Scope=*/FileScope, /*Name=*/FuncName, /*LinkageName=*/StringRef(), + /*File=*/FileScope, /*LineNo=*/0, /*Ty=*/DIFnTy, + /*ScopeLine=*/0, + /*Flags=*/llvm::DINode::FlagArtificial, + /*SPFlags=*/llvm::DISubprogram::SPFlagDefinition, + /*TParams=*/nullptr, /*ThrownTypes=*/nullptr, /*Annotations=*/nullptr); + } + + return SP; +} + void CGDebugInfo::CollectRecordLambdaFields( const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType *RecordTy) { @@ -2751,7 +2877,7 @@ CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { // Collect data fields (including static variables and any initializers). CollectRecordFields(RD, DefUnit, EltTys, FwdDecl); - if (CXXDecl) + if (CXXDecl && !CGM.getCodeGenOpts().DebugOmitUnreferencedMethods) CollectCXXMemberFunctions(CXXDecl, DefUnit, EltTys, FwdDecl); LexicalBlockStack.pop_back(); @@ -3237,7 +3363,7 @@ llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) { // }; int64_t Count = -1; // Count == -1 is an unbounded array. if (const auto *CAT = dyn_cast<ConstantArrayType>(Ty)) - Count = CAT->getSize().getZExtValue(); + Count = CAT->getZExtSize(); else if (const auto *VAT = dyn_cast<VariableArrayType>(Ty)) { if (Expr *Size = VAT->getSizeExpr()) { Expr::EvalResult Result; @@ -3424,6 +3550,23 @@ llvm::DIMacroFile *CGDebugInfo::CreateTempMacroFile(llvm::DIMacroFile *Parent, return DBuilder.createTempMacroFile(Parent, Line, FName); } +llvm::DILocation *CGDebugInfo::CreateTrapFailureMessageFor( + llvm::DebugLoc TrapLocation, StringRef Category, StringRef FailureMsg) { + // Create a debug location from `TrapLocation` that adds an artificial inline + // frame. + SmallString<64> FuncName(ClangTrapPrefix); + + FuncName += "$"; + FuncName += Category; + FuncName += "$"; + FuncName += FailureMsg; + + llvm::DISubprogram *TrapSP = + createInlinedTrapSubprogram(FuncName, TrapLocation->getFile()); + return llvm::DILocation::get(CGM.getLLVMContext(), /*Line=*/0, /*Column=*/0, + /*Scope=*/TrapSP, /*InlinedAt=*/TrapLocation); +} + static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { Qualifiers Quals; do { @@ -3461,6 +3604,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { case Type::BTFTagAttributed: T = cast<BTFTagAttributedType>(T)->getWrappedType(); break; + case Type::CountAttributed: + T = cast<CountAttributedType>(T)->desugar(); + break; case Type::Elaborated: T = cast<ElaboratedType>(T)->getNamedType(); break; @@ -3483,6 +3629,10 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { T = DT; break; } + case Type::PackIndexing: { + T = cast<PackIndexingType>(T)->getSelectedType(); + break; + } case Type::Adjusted: case Type::Decayed: // Decayed and adjusted types use the adjusted type in LLVM and DWARF. @@ -3631,6 +3781,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { case Type::ConstantArray: case Type::VariableArray: case Type::IncompleteArray: + case Type::ArrayParameter: return CreateType(cast<ArrayType>(Ty), Unit); case Type::LValueReference: @@ -3652,6 +3803,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { case Type::TemplateSpecialization: return CreateType(cast<TemplateSpecializationType>(Ty), Unit); + case Type::CountAttributed: case Type::Auto: case Type::Attributed: case Type::BTFTagAttributed: @@ -3666,6 +3818,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { case Type::TypeOfExpr: case Type::TypeOf: case Type::Decltype: + case Type::PackIndexing: case Type::UnaryTransform: break; } @@ -4770,40 +4923,6 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const VarDecl *VD, return D; } -llvm::DIType *CGDebugInfo::CreateBindingDeclType(const BindingDecl *BD) { - llvm::DIFile *Unit = getOrCreateFile(BD->getLocation()); - - // If the declaration is bound to a bitfield struct field, its type may have a - // size that is different from its deduced declaration type's. - if (const MemberExpr *ME = dyn_cast<MemberExpr>(BD->getBinding())) { - if (const FieldDecl *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) { - if (FD->isBitField()) { - ASTContext &Context = CGM.getContext(); - const CGRecordLayout &RL = - CGM.getTypes().getCGRecordLayout(FD->getParent()); - const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD); - - // Find an integer type with the same bitwidth as the bitfield size. If - // no suitable type is present in the target, give up on producing debug - // information as it would be wrong. It is certainly possible to produce - // correct debug info, but the logic isn't currently implemented. - uint64_t BitfieldSizeInBits = Info.Size; - QualType IntTy = - Context.getIntTypeForBitwidth(BitfieldSizeInBits, Info.IsSigned); - if (IntTy.isNull()) - return nullptr; - Qualifiers Quals = BD->getType().getQualifiers(); - QualType FinalTy = Context.getQualifiedType(IntTy, Quals); - llvm::DIType *Ty = getOrCreateType(FinalTy, Unit); - assert(Ty); - return Ty; - } - } - } - - return getOrCreateType(BD->getType(), Unit); -} - llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const BindingDecl *BD, llvm::Value *Storage, std::optional<unsigned> ArgNo, @@ -4818,7 +4937,8 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const BindingDecl *BD, if (isa<DeclRefExpr>(BD->getBinding())) return nullptr; - llvm::DIType *Ty = CreateBindingDeclType(BD); + llvm::DIFile *Unit = getOrCreateFile(BD->getLocation()); + llvm::DIType *Ty = getOrCreateType(BD->getType(), Unit); // If there is no debug info for this type then do not emit debug info // for this variable. @@ -4844,7 +4964,6 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const BindingDecl *BD, unsigned Column = getColumnNumber(BD->getLocation()); StringRef Name = BD->getName(); auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back()); - llvm::DIFile *Unit = getOrCreateFile(BD->getLocation()); // Create the descriptor for the variable. llvm::DILocalVariable *D = DBuilder.createAutoVariable( Scope, Name, Unit, Line, Ty, CGM.getLangOpts().Optimize, @@ -4858,13 +4977,29 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const BindingDecl *BD, const ASTRecordLayout &layout = CGM.getContext().getASTRecordLayout(parent); const uint64_t fieldOffset = layout.getFieldOffset(fieldIndex); - - if (fieldOffset != 0) { - // Currently if the field offset is not a multiple of byte, the produced - // location would not be accurate. Therefore give up. - if (fieldOffset % CGM.getContext().getCharWidth() != 0) - return nullptr; - + if (FD->isBitField()) { + const CGRecordLayout &RL = + CGM.getTypes().getCGRecordLayout(FD->getParent()); + const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD); + // Use DW_OP_plus_uconst to adjust to the start of the bitfield + // storage. + if (!Info.StorageOffset.isZero()) { + Expr.push_back(llvm::dwarf::DW_OP_plus_uconst); + Expr.push_back(Info.StorageOffset.getQuantity()); + } + // Use LLVM_extract_bits to extract the appropriate bits from this + // bitfield. + Expr.push_back(Info.IsSigned + ? llvm::dwarf::DW_OP_LLVM_extract_bits_sext + : llvm::dwarf::DW_OP_LLVM_extract_bits_zext); + Expr.push_back(Info.Offset); + // If we have an oversized bitfield then the value won't be more than + // the size of the type. + const uint64_t TypeSize = CGM.getContext().getTypeSize(BD->getType()); + Expr.push_back(std::min((uint64_t)Info.Size, TypeSize)); + } else if (fieldOffset != 0) { + assert(fieldOffset % CGM.getContext().getCharWidth() == 0 && + "Unexpected non-bitfield with non-byte-aligned offset"); Expr.push_back(llvm::dwarf::DW_OP_plus_uconst); Expr.push_back( CGM.getContext().toCharUnitsFromBits(fieldOffset).getQuantity()); @@ -5350,6 +5485,54 @@ static bool IsReconstitutableType(QualType QT) { return T.Reconstitutable; } +bool CGDebugInfo::HasReconstitutableArgs( + ArrayRef<TemplateArgument> Args) const { + return llvm::all_of(Args, [&](const TemplateArgument &TA) { + switch (TA.getKind()) { + case TemplateArgument::Template: + // Easy to reconstitute - the value of the parameter in the debug + // info is the string name of the template. The template name + // itself won't benefit from any name rebuilding, but that's a + // representational limitation - maybe DWARF could be + // changed/improved to use some more structural representation. + return true; + case TemplateArgument::Declaration: + // Reference and pointer non-type template parameters point to + // variables, functions, etc and their value is, at best (for + // variables) represented as an address - not a reference to the + // DWARF describing the variable/function/etc. This makes it hard, + // possibly impossible to rebuild the original name - looking up + // the address in the executable file's symbol table would be + // needed. + return false; + case TemplateArgument::NullPtr: + // These could be rebuilt, but figured they're close enough to the + // declaration case, and not worth rebuilding. + return false; + case TemplateArgument::Pack: + // A pack is invalid if any of the elements of the pack are + // invalid. + return HasReconstitutableArgs(TA.getPackAsArray()); + case TemplateArgument::Integral: + // Larger integers get encoded as DWARF blocks which are a bit + // harder to parse back into a large integer, etc - so punting on + // this for now. Re-parsing the integers back into APInt is + // probably feasible some day. + return TA.getAsIntegral().getBitWidth() <= 64 && + IsReconstitutableType(TA.getIntegralType()); + case TemplateArgument::StructuralValue: + return false; + case TemplateArgument::Type: + return IsReconstitutableType(TA.getAsType()); + case TemplateArgument::Expression: + return IsReconstitutableType(TA.getAsExpr()->getType()); + default: + llvm_unreachable("Other, unresolved, template arguments should " + "not be seen here"); + } + }); +} + std::string CGDebugInfo::GetName(const Decl *D, bool Qualified) const { std::string Name; llvm::raw_string_ostream OS(Name); @@ -5376,49 +5559,7 @@ std::string CGDebugInfo::GetName(const Decl *D, bool Qualified) const { } else if (auto *VD = dyn_cast<VarDecl>(ND)) { Args = GetTemplateArgs(VD); } - std::function<bool(ArrayRef<TemplateArgument>)> HasReconstitutableArgs = - [&](ArrayRef<TemplateArgument> Args) { - return llvm::all_of(Args, [&](const TemplateArgument &TA) { - switch (TA.getKind()) { - case TemplateArgument::Template: - // Easy to reconstitute - the value of the parameter in the debug - // info is the string name of the template. (so the template name - // itself won't benefit from any name rebuilding, but that's a - // representational limitation - maybe DWARF could be - // changed/improved to use some more structural representation) - return true; - case TemplateArgument::Declaration: - // Reference and pointer non-type template parameters point to - // variables, functions, etc and their value is, at best (for - // variables) represented as an address - not a reference to the - // DWARF describing the variable/function/etc. This makes it hard, - // possibly impossible to rebuild the original name - looking up the - // address in the executable file's symbol table would be needed. - return false; - case TemplateArgument::NullPtr: - // These could be rebuilt, but figured they're close enough to the - // declaration case, and not worth rebuilding. - return false; - case TemplateArgument::Pack: - // A pack is invalid if any of the elements of the pack are invalid. - return HasReconstitutableArgs(TA.getPackAsArray()); - case TemplateArgument::Integral: - // Larger integers get encoded as DWARF blocks which are a bit - // harder to parse back into a large integer, etc - so punting on - // this for now. Re-parsing the integers back into APInt is probably - // feasible some day. - return TA.getAsIntegral().getBitWidth() <= 64 && - IsReconstitutableType(TA.getIntegralType()); - case TemplateArgument::StructuralValue: - return false; - case TemplateArgument::Type: - return IsReconstitutableType(TA.getAsType()); - default: - llvm_unreachable("Other, unresolved, template arguments should " - "not be seen here"); - } - }); - }; + // A conversion operator presents complications/ambiguity if there's a // conversion to class template that is itself a template, eg: // template<typename T> @@ -5636,6 +5777,48 @@ void CGDebugInfo::EmitExternalVariable(llvm::GlobalVariable *Var, Var->addDebugInfo(GVE); } +void CGDebugInfo::EmitPseudoVariable(CGBuilderTy &Builder, + llvm::Instruction *Value, QualType Ty) { + // Only when -g2 or above is specified, debug info for variables will be + // generated. + if (CGM.getCodeGenOpts().getDebugInfo() <= + llvm::codegenoptions::DebugLineTablesOnly) + return; + + llvm::DILocation *DIL = Value->getDebugLoc().get(); + if (!DIL) + return; + + llvm::DIFile *Unit = DIL->getFile(); + llvm::DIType *Type = getOrCreateType(Ty, Unit); + + // Check if Value is already a declared variable and has debug info, in this + // case we have nothing to do. Clang emits a declared variable as alloca, and + // it is loaded upon use, so we identify such pattern here. + if (llvm::LoadInst *Load = dyn_cast<llvm::LoadInst>(Value)) { + llvm::Value *Var = Load->getPointerOperand(); + // There can be implicit type cast applied on a variable if it is an opaque + // ptr, in this case its debug info may not match the actual type of object + // being used as in the next instruction, so we will need to emit a pseudo + // variable for type-casted value. + auto DeclareTypeMatches = [&](auto *DbgDeclare) { + return DbgDeclare->getVariable()->getType() == Type; + }; + if (any_of(llvm::findDbgDeclares(Var), DeclareTypeMatches) || + any_of(llvm::findDVRDeclares(Var), DeclareTypeMatches)) + return; + } + + llvm::DILocalVariable *D = + DBuilder.createAutoVariable(LexicalBlockStack.back(), "", nullptr, 0, + Type, false, llvm::DINode::FlagArtificial); + + if (auto InsertPoint = Value->getInsertionPointAfterDef()) { + DBuilder.insertDbgValueIntrinsic(Value, D, DBuilder.createExpression(), DIL, + &**InsertPoint); + } +} + void CGDebugInfo::EmitGlobalAlias(const llvm::GlobalValue *GV, const GlobalDecl GD) { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.h b/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.h index 7b60e94555d0..a0c419cf1e20 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGDebugInfo.h @@ -20,8 +20,8 @@ #include "clang/AST/PrettyPrinter.h" #include "clang/AST/Type.h" #include "clang/AST/TypeOrdering.h" +#include "clang/Basic/ASTSourceDescriptor.h" #include "clang/Basic/CodeGenOptions.h" -#include "clang/Basic/Module.h" #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" @@ -29,7 +29,9 @@ #include "llvm/IR/DebugInfo.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Allocator.h" +#include <map> #include <optional> +#include <string> namespace llvm { class MDNode; @@ -38,6 +40,7 @@ class MDNode; namespace clang { class ClassTemplateSpecializationDecl; class GlobalDecl; +class Module; class ModuleMap; class ObjCInterfaceDecl; class UsingDecl; @@ -82,6 +85,8 @@ class CGDebugInfo { #include "clang/Basic/OpenCLExtensionTypes.def" #define WASM_TYPE(Name, Id, SingletonId) llvm::DIType *SingletonId = nullptr; #include "clang/Basic/WebAssemblyReferenceTypes.def" +#define AMDGPU_TYPE(Name, Id, SingletonId) llvm::DIType *SingletonId = nullptr; +#include "clang/Basic/AMDGPUTypes.def" /// Cache of previously constructed Types. llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache; @@ -337,15 +342,20 @@ class CGDebugInfo { llvm::DIScope *RecordTy, const RecordDecl *RD); - /// Create type for binding declarations. - llvm::DIType *CreateBindingDeclType(const BindingDecl *BD); - /// Create an anonnymous zero-size separator for bit-field-decl if needed on /// the target. llvm::DIDerivedType *createBitFieldSeparatorIfNeeded( const FieldDecl *BitFieldDecl, const llvm::DIDerivedType *BitFieldDI, llvm::ArrayRef<llvm::Metadata *> PreviousFieldsDI, const RecordDecl *RD); + /// A cache that maps names of artificial inlined functions to subprograms. + llvm::StringMap<llvm::DISubprogram *> InlinedTrapFuncMap; + + /// A function that returns the subprogram corresponding to the artificial + /// inlined function for traps. + llvm::DISubprogram *createInlinedTrapSubprogram(StringRef FuncName, + llvm::DIFile *FileScope); + /// Helpers for collecting fields of a record. /// @{ void CollectRecordLambdaFields(const CXXRecordDecl *CXXDecl, @@ -529,6 +539,12 @@ public: /// Emit information about an external variable. void EmitExternalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl); + /// Emit a pseudo variable and debug info for an intermediate value if it does + /// not correspond to a variable in the source code, so that a profiler can + /// track more accurate usage of certain instructions of interest. + void EmitPseudoVariable(CGBuilderTy &Builder, llvm::Instruction *Value, + QualType Ty); + /// Emit information about global variable alias. void EmitGlobalAlias(const llvm::GlobalValue *GV, const GlobalDecl Decl); @@ -602,6 +618,18 @@ public: return CoroutineParameterMappings; } + /// Create a debug location from `TrapLocation` that adds an artificial inline + /// frame where the frame name is + /// + /// * `<Prefix>:<Category>:<FailureMsg>` + /// + /// `<Prefix>` is "__clang_trap_msg". + /// + /// This is used to store failure reasons for traps. + llvm::DILocation *CreateTrapFailureMessageFor(llvm::DebugLoc TrapLocation, + StringRef Category, + StringRef FailureMsg); + private: /// Emit call to llvm.dbg.declare for a variable declaration. /// Returns a pointer to the DILocalVariable associated with the @@ -626,7 +654,8 @@ private: llvm::DIType *WrappedType; }; - std::string GetName(const Decl*, bool Qualified = false) const; + bool HasReconstitutableArgs(ArrayRef<TemplateArgument> Args) const; + std::string GetName(const Decl *, bool Qualified = false) const; /// Build up structure info for the byref. See \a BuildByRefType. BlockByRefType EmitTypeForVarWithBlocksAttr(const VarDecl *VD, diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGDecl.cpp index aa9997b87ecf..c3251bb5ab56 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGDecl.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGDecl.cpp @@ -19,6 +19,7 @@ #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "ConstantEmitter.h" +#include "EHScopeStack.h" #include "PatternInit.h" #include "TargetInfo.h" #include "clang/AST/ASTContext.h" @@ -32,9 +33,11 @@ #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Sema/Sema.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Type.h" #include <optional> @@ -284,6 +287,7 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl( setTLSMode(GV, D); setGVProperties(GV, &D); + getTargetCodeGenInfo().setTargetAttributes(cast<Decl>(&D), GV, *this); // Make sure the result is of the correct type. LangAS ExpectedAS = Ty.getAddressSpace(); @@ -735,18 +739,17 @@ static bool tryEmitARCCopyWeakInit(CodeGenFunction &CGF, LValue srcLV = CGF.EmitLValue(srcExpr); // Handle a formal type change to avoid asserting. - auto srcAddr = srcLV.getAddress(CGF); + auto srcAddr = srcLV.getAddress(); if (needsCast) { - srcAddr = - srcAddr.withElementType(destLV.getAddress(CGF).getElementType()); + srcAddr = srcAddr.withElementType(destLV.getAddress().getElementType()); } // If it was an l-value, use objc_copyWeak. if (srcExpr->isLValue()) { - CGF.EmitARCCopyWeak(destLV.getAddress(CGF), srcAddr); + CGF.EmitARCCopyWeak(destLV.getAddress(), srcAddr); } else { assert(srcExpr->isXValue()); - CGF.EmitARCMoveWeak(destLV.getAddress(CGF), srcAddr); + CGF.EmitARCMoveWeak(destLV.getAddress(), srcAddr); } return true; } @@ -764,7 +767,7 @@ static bool tryEmitARCCopyWeakInit(CodeGenFunction &CGF, static void drillIntoBlockVariable(CodeGenFunction &CGF, LValue &lvalue, const VarDecl *var) { - lvalue.setAddress(CGF.emitBlockByrefAddress(lvalue.getAddress(CGF), var)); + lvalue.setAddress(CGF.emitBlockByrefAddress(lvalue.getAddress(), var)); } void CodeGenFunction::EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, @@ -823,18 +826,17 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, if (capturedByInit) { // We can use a simple GEP for this because it can't have been // moved yet. - tempLV.setAddress(emitBlockByrefAddress(tempLV.getAddress(*this), + tempLV.setAddress(emitBlockByrefAddress(tempLV.getAddress(), cast<VarDecl>(D), /*follow*/ false)); } - auto ty = - cast<llvm::PointerType>(tempLV.getAddress(*this).getElementType()); + auto ty = cast<llvm::PointerType>(tempLV.getAddress().getElementType()); llvm::Value *zero = CGM.getNullPointer(ty, tempLV.getType()); // If __weak, we want to use a barrier under certain conditions. if (lifetime == Qualifiers::OCL_Weak) - EmitARCInitWeak(tempLV.getAddress(*this), zero); + EmitARCInitWeak(tempLV.getAddress(), zero); // Otherwise just do a simple store. else @@ -877,9 +879,9 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, if (capturedByInit) drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D)); if (accessedByInit) - EmitARCStoreWeak(lvalue.getAddress(*this), value, /*ignored*/ true); + EmitARCStoreWeak(lvalue.getAddress(), value, /*ignored*/ true); else - EmitARCInitWeak(lvalue.getAddress(*this), value); + EmitARCInitWeak(lvalue.getAddress(), value); return; } @@ -1382,7 +1384,7 @@ void CodeGenFunction::EmitAndRegisterVariableArrayDimensions( // For each dimension stores its QualType and corresponding // size-expression Value. SmallVector<CodeGenFunction::VlaSizePair, 4> Dimensions; - SmallVector<IdentifierInfo *, 4> VLAExprNames; + SmallVector<const IdentifierInfo *, 4> VLAExprNames; // Break down the array into individual dimensions. QualType Type1D = D.getType(); @@ -1419,7 +1421,7 @@ void CodeGenFunction::EmitAndRegisterVariableArrayDimensions( MD = llvm::ConstantAsMetadata::get(C); else { // Create an artificial VarDecl to generate debug info for. - IdentifierInfo *NameIdent = VLAExprNames[NameIdx++]; + const IdentifierInfo *NameIdent = VLAExprNames[NameIdx++]; auto QT = getContext().getIntTypeForBitwidth( SizeTy->getScalarSizeInBits(), false); auto *ArtificialDecl = VarDecl::Create( @@ -1460,7 +1462,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo(); Address address = Address::invalid(); - Address AllocaAddr = Address::invalid(); + RawAddress AllocaAddr = RawAddress::invalid(); Address OpenMPLocalAddr = Address::invalid(); if (CGM.getLangOpts().OpenMPIRBuilder) OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D); @@ -1523,7 +1525,10 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // return slot, so that we can elide the copy when returning this // variable (C++0x [class.copy]p34). address = ReturnValue; - AllocaAddr = ReturnValue; + AllocaAddr = + RawAddress(ReturnValue.emitRawPointer(*this), + ReturnValue.getElementType(), ReturnValue.getAlignment()); + ; if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { const auto *RD = RecordTy->getDecl(); @@ -1534,7 +1539,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // to this variable. Set it to zero to indicate that NRVO was not // applied. llvm::Value *Zero = Builder.getFalse(); - Address NRVOFlag = + RawAddress NRVOFlag = CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo"); EnsureInsertPoint(); Builder.CreateStore(Zero, NRVOFlag); @@ -1614,7 +1619,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { LValue Base = MakeAddrLValue(AddrSizePair.first, D.getType(), CGM.getContext().getDeclAlign(&D), AlignmentSource::Decl); - address = Base.getAddress(*this); + address = Base.getAddress(); // Push a cleanup block to emit the call to __kmpc_free_shared in the // appropriate location at the end of the scope of the @@ -1677,7 +1682,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { } if (D.hasAttr<AnnotateAttr>() && HaveInsertPoint()) - EmitVarAnnotations(&D, address.getPointer()); + EmitVarAnnotations(&D, address.emitRawPointer(*this)); // Make sure we call @llvm.lifetime.end. if (emission.useLifetimeMarkers()) @@ -1850,12 +1855,13 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type, llvm::Value *BaseSizeInChars = llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity()); Address Begin = Loc.withElementType(Int8Ty); - llvm::Value *End = Builder.CreateInBoundsGEP( - Begin.getElementType(), Begin.getPointer(), SizeVal, "vla.end"); + llvm::Value *End = Builder.CreateInBoundsGEP(Begin.getElementType(), + Begin.emitRawPointer(*this), + SizeVal, "vla.end"); llvm::BasicBlock *OriginBB = Builder.GetInsertBlock(); EmitBlock(LoopBB); llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur"); - Cur->addIncoming(Begin.getPointer(), OriginBB); + Cur->addIncoming(Begin.emitRawPointer(*this), OriginBB); CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize); auto *I = Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign), @@ -1964,10 +1970,35 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { constant = constWithPadding(CGM, IsPattern::No, replaceUndef(CGM, isPattern, constant)); } + + if (D.getType()->isBitIntType() && + CGM.getTypes().typeRequiresSplitIntoByteArray(D.getType())) { + // Constants for long _BitInt types are split into individual bytes. + // Try to fold these back into an integer constant so it can be stored + // properly. + llvm::Type *LoadType = CGM.getTypes().convertTypeForLoadStore( + D.getType(), constant->getType()); + constant = llvm::ConstantFoldLoadFromConst( + constant, LoadType, llvm::APInt::getZero(32), CGM.getDataLayout()); + } } if (!constant) { - initializeWhatIsTechnicallyUninitialized(Loc); + if (trivialAutoVarInit != + LangOptions::TrivialAutoVarInitKind::Uninitialized) { + // At this point, we know D has an Init expression, but isn't a constant. + // - If D is not a scalar, auto-var-init conservatively (members may be + // left uninitialized by constructor Init expressions for example). + // - If D is a scalar, we only need to auto-var-init if there is a + // self-reference. Otherwise, the Init expression should be sufficient. + // It may be that the Init expression uses other uninitialized memory, + // but auto-var-init here would not help, as auto-init would get + // overwritten by Init. + if (!D.getType()->isScalarType() || capturedByInit || + isAccessedBy(D, Init)) { + initializeWhatIsTechnicallyUninitialized(Loc); + } + } LValue lv = MakeAddrLValue(Loc, type); lv.setNonGC(true); return EmitExprAsInit(Init, &D, lv, capturedByInit); @@ -2027,10 +2058,10 @@ void CodeGenFunction::EmitExprAsInit(const Expr *init, const ValueDecl *D, else if (auto *FD = dyn_cast<FieldDecl>(D)) Overlap = getOverlapForFieldInit(FD); // TODO: how can we delay here if D is captured by its initializer? - EmitAggExpr(init, AggValueSlot::forLValue( - lvalue, *this, AggValueSlot::IsDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased, Overlap)); + EmitAggExpr(init, + AggValueSlot::forLValue(lvalue, AggValueSlot::IsDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, Overlap)); } return; } @@ -2196,6 +2227,27 @@ void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr, destroyer, useEHCleanupForArray); } +// Pushes a destroy and defers its deactivation until its +// CleanupDeactivationScope is exited. +void CodeGenFunction::pushDestroyAndDeferDeactivation( + QualType::DestructionKind dtorKind, Address addr, QualType type) { + assert(dtorKind && "cannot push destructor for trivial type"); + + CleanupKind cleanupKind = getCleanupKind(dtorKind); + pushDestroyAndDeferDeactivation( + cleanupKind, addr, type, getDestroyer(dtorKind), cleanupKind & EHCleanup); +} + +void CodeGenFunction::pushDestroyAndDeferDeactivation( + CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer, + bool useEHCleanupForArray) { + llvm::Instruction *DominatingIP = + Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); + pushDestroy(cleanupKind, addr, type, destroyer, useEHCleanupForArray); + DeferredDeactivationCleanupStack.push_back( + {EHStack.stable_begin(), DominatingIP}); +} + void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) { EHStack.pushCleanup<CallStackRestore>(Kind, SPMem); } @@ -2212,39 +2264,48 @@ void CodeGenFunction::pushLifetimeExtendedDestroy(CleanupKind cleanupKind, // If we're not in a conditional branch, we don't need to bother generating a // conditional cleanup. if (!isInConditionalBranch()) { - // Push an EH-only cleanup for the object now. // FIXME: When popping normal cleanups, we need to keep this EH cleanup // around in case a temporary's destructor throws an exception. - if (cleanupKind & EHCleanup) - EHStack.pushCleanup<DestroyObject>( - static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), addr, type, - destroyer, useEHCleanupForArray); + // Add the cleanup to the EHStack. After the full-expr, this would be + // deactivated before being popped from the stack. + pushDestroyAndDeferDeactivation(cleanupKind, addr, type, destroyer, + useEHCleanupForArray); + + // Since this is lifetime-extended, push it once again to the EHStack after + // the full expression. return pushCleanupAfterFullExprWithActiveFlag<DestroyObject>( - cleanupKind, Address::invalid(), addr, type, destroyer, useEHCleanupForArray); + cleanupKind, Address::invalid(), addr, type, destroyer, + useEHCleanupForArray); } // Otherwise, we should only destroy the object if it's been initialized. - // Re-use the active flag and saved address across both the EH and end of - // scope cleanups. - using SavedType = typename DominatingValue<Address>::saved_type; using ConditionalCleanupType = EHScopeStack::ConditionalCleanup<DestroyObject, Address, QualType, Destroyer *, bool>; - - Address ActiveFlag = createCleanupActiveFlag(); - SavedType SavedAddr = saveValueInCond(addr); - - if (cleanupKind & EHCleanup) { - EHStack.pushCleanup<ConditionalCleanupType>( - static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), SavedAddr, type, - destroyer, useEHCleanupForArray); - initFullExprCleanupWithFlag(ActiveFlag); - } - + DominatingValue<Address>::saved_type SavedAddr = saveValueInCond(addr); + + // Remember to emit cleanup if we branch-out before end of full-expression + // (eg: through stmt-expr or coro suspensions). + AllocaTrackerRAII DeactivationAllocas(*this); + Address ActiveFlagForDeactivation = createCleanupActiveFlag(); + + pushCleanupAndDeferDeactivation<ConditionalCleanupType>( + cleanupKind, SavedAddr, type, destroyer, useEHCleanupForArray); + initFullExprCleanupWithFlag(ActiveFlagForDeactivation); + EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin()); + // Erase the active flag if the cleanup was not emitted. + cleanup.AddAuxAllocas(std::move(DeactivationAllocas).Take()); + + // Since this is lifetime-extended, push it once again to the EHStack after + // the full expression. + // The previous active flag would always be 'false' due to forced deferred + // deactivation. Use a separate flag for lifetime-extension to correctly + // remember if this branch was taken and the object was initialized. + Address ActiveFlagForLifetimeExt = createCleanupActiveFlag(); pushCleanupAfterFullExprWithActiveFlag<ConditionalCleanupType>( - cleanupKind, ActiveFlag, SavedAddr, type, destroyer, + cleanupKind, ActiveFlagForLifetimeExt, SavedAddr, type, destroyer, useEHCleanupForArray); } @@ -2282,7 +2343,7 @@ void CodeGenFunction::emitDestroy(Address addr, QualType type, checkZeroLength = false; } - llvm::Value *begin = addr.getPointer(); + llvm::Value *begin = addr.emitRawPointer(*this); llvm::Value *end = Builder.CreateInBoundsGEP(addr.getElementType(), begin, length); emitArrayDestroy(begin, end, type, elementAlign, destroyer, @@ -2437,9 +2498,9 @@ namespace { }; } // end anonymous namespace -/// pushIrregularPartialArrayCleanup - Push an EH cleanup to destroy -/// already-constructed elements of the given array. The cleanup -/// may be popped with DeactivateCleanupBlock or PopCleanupBlock. +/// pushIrregularPartialArrayCleanup - Push a NormalAndEHCleanup to +/// destroy already-constructed elements of the given array. The cleanup may be +/// popped with DeactivateCleanupBlock or PopCleanupBlock. /// /// \param elementType - the immediate element type of the array; /// possibly still an array type @@ -2448,10 +2509,9 @@ void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin, QualType elementType, CharUnits elementAlign, Destroyer *destroyer) { - pushFullExprCleanup<IrregularPartialArrayDestroy>(EHCleanup, - arrayBegin, arrayEndPointer, - elementType, elementAlign, - destroyer); + pushFullExprCleanup<IrregularPartialArrayDestroy>( + NormalAndEHCleanup, arrayBegin, arrayEndPointer, elementType, + elementAlign, destroyer); } /// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy @@ -2542,7 +2602,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } Address DeclPtr = Address::invalid(); - Address AllocaPtr = Address::invalid(); + RawAddress AllocaPtr = Address::invalid(); bool DoStore = false; bool IsScalar = hasScalarEvaluationKind(Ty); bool UseIndirectDebugAddress = false; @@ -2554,8 +2614,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, // Indirect argument is in alloca address space, which may be different // from the default address space. auto AllocaAS = CGM.getASTAllocaAddressSpace(); - auto *V = DeclPtr.getPointer(); - AllocaPtr = DeclPtr; + auto *V = DeclPtr.emitRawPointer(*this); + AllocaPtr = RawAddress(V, DeclPtr.getElementType(), DeclPtr.getAlignment()); // For truly ABI indirect arguments -- those that are not `byval` -- store // the address of the argument on the stack to preserve debug information. @@ -2647,7 +2707,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, // objc_storeStrong attempts to release its old value. llvm::Value *Null = CGM.EmitNullConstant(D.getType()); EmitStoreOfScalar(Null, lv, /* isInitialization */ true); - EmitARCStoreStrongCall(lv.getAddress(*this), ArgVal, true); + EmitARCStoreStrongCall(lv.getAddress(), ArgVal, true); DoStore = false; } else @@ -2694,7 +2754,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } if (D.hasAttr<AnnotateAttr>()) - EmitVarAnnotations(&D, DeclPtr.getPointer()); + EmitVarAnnotations(&D, DeclPtr.emitRawPointer(*this)); // We can only check return value nullability if all arguments to the // function satisfy their nullability preconditions. This makes it necessary diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGDeclCXX.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGDeclCXX.cpp index e08a1e5f42df..2f56355cff90 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGDeclCXX.cpp @@ -57,7 +57,7 @@ static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D, return; case TEK_Aggregate: CGF.EmitAggExpr(Init, - AggValueSlot::forLValue(lv, CGF, AggValueSlot::IsDestructed, + AggValueSlot::forLValue(lv, AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap)); @@ -162,7 +162,8 @@ void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) { // Grab the llvm.invariant.start intrinsic. llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; // Overloaded address space type. - llvm::Type *ObjectPtr[1] = {Int8PtrTy}; + assert(Addr->getType()->isPointerTy() && "Address must be a pointer"); + llvm::Type *ObjectPtr[1] = {Addr->getType()}; llvm::Function *InvariantStart = CGM.getIntrinsic(InvStartID, ObjectPtr); // Emit a call with the size in bytes of the object. @@ -231,7 +232,7 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, /// Create a stub function, suitable for being passed to atexit, /// which passes the given address to the given destructor function. -llvm::Function *CodeGenFunction::createAtExitStub(const VarDecl &VD, +llvm::Constant *CodeGenFunction::createAtExitStub(const VarDecl &VD, llvm::FunctionCallee dtor, llvm::Constant *addr) { // Get the destructor function type, void(*)(void). @@ -263,7 +264,12 @@ llvm::Function *CodeGenFunction::createAtExitStub(const VarDecl &VD, CGF.FinishFunction(); - return fn; + // Get a proper function pointer. + FunctionProtoType::ExtProtoInfo EPI(getContext().getDefaultCallingConvention( + /*IsVariadic=*/false, /*IsCXXMethod=*/false)); + QualType fnType = getContext().getFunctionType(getContext().VoidTy, + {getContext().VoidPtrTy}, EPI); + return CGM.getFunctionPointer(fn, fnType); } /// Create a stub function, suitable for being passed to __pt_atexit_np, @@ -332,7 +338,8 @@ void CodeGenFunction::registerGlobalDtorWithLLVM(const VarDecl &VD, llvm::FunctionCallee Dtor, llvm::Constant *Addr) { // Create a function which calls the destructor. - llvm::Function *dtorStub = createAtExitStub(VD, Dtor, Addr); + llvm::Function *dtorStub = + cast<llvm::Function>(createAtExitStub(VD, Dtor, Addr)); CGM.AddGlobalDtor(dtorStub); } @@ -476,6 +483,10 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction( !isInNoSanitizeList(SanitizerKind::Thread, Fn, Loc)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); + if (getLangOpts().Sanitize.has(SanitizerKind::NumericalStability) && + !isInNoSanitizeList(SanitizerKind::NumericalStability, Fn, Loc)) + Fn->addFnAttr(llvm::Attribute::SanitizeNumericalStability); + if (getLangOpts().Sanitize.has(SanitizerKind::Memory) && !isInNoSanitizeList(SanitizerKind::Memory, Fn, Loc)) Fn->addFnAttr(llvm::Attribute::SanitizeMemory); @@ -835,6 +846,10 @@ CodeGenModule::EmitCXXGlobalInitFunc() { // No Itanium initializer in header like modules. if (M->isHeaderLikeModule()) continue; + // We're allowed to skip the initialization if we are sure it doesn't + // do any thing. + if (!M->isNamedModuleInterfaceHasInit()) + continue; llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); SmallString<256> FnName; { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGException.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGException.cpp index 5a9d06da12de..bb2ed237ee9f 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGException.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGException.cpp @@ -397,7 +397,7 @@ namespace { void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - pushFullExprCleanup<FreeException>(EHCleanup, addr.getPointer()); + pushFullExprCleanup<FreeException>(EHCleanup, addr.emitRawPointer(*this)); EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this @@ -416,8 +416,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { /*IsInit*/ true); // Deactivate the cleanup block. - DeactivateCleanupBlock(cleanup, - cast<llvm::Instruction>(typedAddr.getPointer())); + DeactivateCleanupBlock( + cleanup, cast<llvm::Instruction>(typedAddr.emitRawPointer(*this))); } Address CodeGenFunction::getExceptionSlot() { @@ -1052,7 +1052,8 @@ static void emitWasmCatchPadBlock(CodeGenFunction &CGF, CGF.Builder.CreateStore(Exn, CGF.getExceptionSlot()); llvm::CallInst *Selector = CGF.Builder.CreateCall(GetSelectorFn, CPI); - llvm::Function *TypeIDFn = CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for); + llvm::Function *TypeIDFn = + CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for, {CGF.VoidPtrTy}); // If there's only a single catch-all, branch directly to its handler. if (CatchScope.getNumHandlers() == 1 && @@ -1137,7 +1138,7 @@ static void emitCatchDispatchBlock(CodeGenFunction &CGF, // Select the right handler. llvm::Function *llvm_eh_typeid_for = - CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for); + CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for, {CGF.VoidPtrTy}); llvm::Type *argTy = llvm_eh_typeid_for->getArg(0)->getType(); LangAS globAS = CGF.CGM.GetGlobalVarAddressSpace(nullptr); @@ -1834,7 +1835,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, llvm::Value *ParentFP) { llvm::CallInst *RecoverCall = nullptr; CGBuilderTy Builder(*this, AllocaInsertPt); - if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar.getPointer())) { + if (auto *ParentAlloca = + dyn_cast_or_null<llvm::AllocaInst>(ParentVar.getBasePointer())) { // Mark the variable escaped if nobody else referenced it and compute the // localescape index. auto InsertPair = ParentCGF.EscapedLocals.insert( @@ -1851,8 +1853,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, // If the parent didn't have an alloca, we're doing some nested outlining. // Just clone the existing localrecover call, but tweak the FP argument to // use our FP value. All other arguments are constants. - auto *ParentRecover = - cast<llvm::IntrinsicInst>(ParentVar.getPointer()->stripPointerCasts()); + auto *ParentRecover = cast<llvm::IntrinsicInst>( + ParentVar.emitRawPointer(*this)->stripPointerCasts()); assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover && "expected alloca or localrecover in parent LocalDeclMap"); RecoverCall = cast<llvm::CallInst>(ParentRecover->clone()); @@ -1925,7 +1927,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, if (isa<ImplicitParamDecl>(D) && D->getType() == getContext().VoidPtrTy) { assert(D->getName().starts_with("frame_pointer")); - FramePtrAddrAlloca = cast<llvm::AllocaInst>(I.second.getPointer()); + FramePtrAddrAlloca = + cast<llvm::AllocaInst>(I.second.getBasePointer()); break; } } @@ -1986,7 +1989,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); + CXXThisValue = ThisFieldLValue.getAddress().emitRawPointer(*this); } else { CXXThisValue = EmitLoadOfLValue(ThisFieldLValue, SourceLocation()) .getScalarVal(); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp index f8f997909977..3ef22b17f769 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "ABIInfoImpl.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" #include "CGCall.h" @@ -56,8 +57,13 @@ using namespace CodeGen; // Experiment to make sanitizers easier to debug static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization( "ubsan-unique-traps", llvm::cl::Optional, - llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check"), - llvm::cl::init(false)); + llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check.")); + +// TODO: Introduce frontend options to enabled per sanitizers, similar to +// `fsanitize-trap`. +static llvm::cl::opt<bool> ClSanitizeGuardChecks( + "ubsan-guard-checks", llvm::cl::Optional, + llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`.")); //===--------------------------------------------------------------------===// // Miscellaneous Helper Methods @@ -65,21 +71,21 @@ static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization( /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. -Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, - CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize) { +RawAddress +CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return Address(Alloca, Ty, Align, KnownNonNull); + return RawAddress(Alloca, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. The alloca is casted to default address space if necessary. -Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize, - Address *AllocaAddr) { +RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize, + RawAddress *AllocaAddr) { auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); if (AllocaAddr) *AllocaAddr = Alloca; @@ -101,7 +107,7 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return Address(V, Ty, Align, KnownNonNull); + return RawAddress(V, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -110,38 +116,45 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, const Twine &Name, llvm::Value *ArraySize) { + llvm::AllocaInst *Alloca; if (ArraySize) - return Builder.CreateAlloca(Ty, ArraySize, Name); - return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(), - ArraySize, Name, AllocaInsertPt); + Alloca = Builder.CreateAlloca(Ty, ArraySize, Name); + else + Alloca = new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(), + ArraySize, Name, &*AllocaInsertPt); + if (Allocas) { + Allocas->Add(Alloca); + } + return Alloca; } /// CreateDefaultAlignTempAlloca - This creates an alloca with the /// default alignment of the corresponding LLVM type, which is *not* /// guaranteed to be related in any way to the expected alignment of /// an AST type that might have been lowered to Ty. -Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name) { +RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name) { CharUnits Align = CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty)); return CreateTempAlloca(Ty, Align, Name); } -Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { +RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { CharUnits Align = getContext().getTypeAlignInChars(Ty); return CreateTempAlloca(ConvertType(Ty), Align, Name); } -Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, - Address *Alloca) { +RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, + RawAddress *Alloca) { // FIXME: Should we prefer the preferred type alignment here? return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca); } -Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, - const Twine &Name, Address *Alloca) { - Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, - /*ArraySize=*/nullptr, Alloca); +RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, + const Twine &Name, + RawAddress *Alloca) { + RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, + /*ArraySize=*/nullptr, Alloca); if (Ty->isConstantMatrixType()) { auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType()); @@ -154,13 +167,14 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, return Result; } -Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, - const Twine &Name) { +RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + CharUnits Align, + const Twine &Name) { return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); } -Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, - const Twine &Name) { +RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + const Twine &Name) { return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty), Name); } @@ -304,8 +318,8 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, CleanupKind CleanupKind; if (Lifetime == Qualifiers::OCL_Strong) { const ValueDecl *VD = M->getExtendingDecl(); - bool Precise = - VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>(); + bool Precise = isa_and_nonnull<VarDecl>(VD) && + VD->hasAttr<ObjCPreciseLifetimeAttr>(); CleanupKind = CGF.getARCCleanupKind(); Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise : &CodeGenFunction::destroyARCStrongImprecise; @@ -359,7 +373,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } else { CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor( GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)); - CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer()); + CleanupArg = cast<llvm::Constant>(ReferenceTemporary.emitRawPointer(CGF)); } CGF.CGM.getCXXABI().registerGlobalDtor( CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg); @@ -384,10 +398,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } } -static Address createReferenceTemporary(CodeGenFunction &CGF, - const MaterializeTemporaryExpr *M, - const Expr *Inner, - Address *Alloca = nullptr) { +static RawAddress createReferenceTemporary(CodeGenFunction &CGF, + const MaterializeTemporaryExpr *M, + const Expr *Inner, + RawAddress *Alloca = nullptr) { auto &TCG = CGF.getTargetHooks(); switch (M->getStorageDuration()) { case SD_FullExpression: @@ -416,7 +430,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, GV->getValueType()->getPointerTo( CGF.getContext().getTargetAddressSpace(LangAS::Default))); // FIXME: Should we put the new global into a COMDAT? - return Address(C, GV->getValueType(), alignment); + return RawAddress(C, GV->getValueType(), alignment); } return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); } @@ -448,7 +462,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { auto ownership = M->getType().getObjCLifetime(); if (ownership != Qualifiers::OCL_None && ownership != Qualifiers::OCL_ExplicitNone) { - Address Object = createReferenceTemporary(*this, M, E); + RawAddress Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { llvm::Type *Ty = ConvertTypeForMem(E->getType()); Object = Object.withElementType(Ty); @@ -502,8 +516,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { } // Create and initialize the reference temporary. - Address Alloca = Address::invalid(); - Address Object = createReferenceTemporary(*this, M, E, &Alloca); + RawAddress Alloca = Address::invalid(); + RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca); if (auto *Var = dyn_cast<llvm::GlobalVariable>( Object.getPointer()->stripPointerCasts())) { llvm::Type *TemporaryType = ConvertTypeForMem(E->getType()); @@ -592,7 +606,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { LV = EmitLValueForField(LV, Adjustment.Field); assert(LV.isSimple() && "materialized temporary field is not a simple lvalue"); - Object = LV.getAddress(*this); + Object = LV.getAddress(); break; } @@ -637,16 +651,13 @@ unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx, ->getZExtValue(); } -/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h. -static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low, - llvm::Value *High) { - llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL); - llvm::Value *K47 = Builder.getInt64(47); - llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul); - llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0); - llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul); - llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0); - return Builder.CreateMul(B1, KMul); +static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc, + llvm::Value *Ptr) { + llvm::Value *A0 = + Builder.CreateMul(Ptr, Builder.getInt64(0xbf58476d1ce4e5b9u)); + llvm::Value *A1 = + Builder.CreateXor(A0, Builder.CreateLShr(A0, Builder.getInt64(31))); + return Builder.CreateXor(Acc, A1); } bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) { @@ -808,11 +819,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, EmitBlock(VptrNotNull); } - // Compute a hash of the mangled name of the type. - // - // FIXME: This is not guaranteed to be deterministic! Move to a - // fingerprinting mechanism once LLVM provides one. For the time - // being the implementation happens to be deterministic. + // Compute a deterministic hash of the mangled name of the type. SmallString<64> MangledName; llvm::raw_svector_ostream Out(MangledName); CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(), @@ -821,15 +828,19 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, // Contained in NoSanitizeList based on the mangled type. if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr, Out.str())) { - llvm::hash_code TypeHash = hash_value(Out.str()); + // Load the vptr, and mix it with TypeHash. + llvm::Value *TypeHash = + llvm::ConstantInt::get(Int64Ty, xxh3_64bits(Out.str())); - // Load the vptr, and compute hash_16_bytes(TypeHash, vptr). - llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash); + llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0); Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign()); - llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr); - llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty); + llvm::Value *VPtrVal = GetVTablePtr(VPtrAddr, VPtrTy, + Ty->getAsCXXRecordDecl(), + VTableAuthMode::UnsafeUbsanStrip); + VPtrVal = Builder.CreateBitOrPointerCast(VPtrVal, IntPtrTy); - llvm::Value *Hash = emitHash16Bytes(Builder, Low, High); + llvm::Value *Hash = + emitHashMix(Builder, TypeHash, Builder.CreateZExt(VPtrVal, Int64Ty)); Hash = Builder.CreateTrunc(Hash, IntPtrTy); // Look the hash up in our cache. @@ -1041,6 +1052,8 @@ public: return Visit(E->getBase()); } const Expr *VisitCastExpr(const CastExpr *E) { + if (E->getCastKind() == CK_LValueToRValue) + return IsExpectedRecordDecl(E) ? E : nullptr; return Visit(E->getSubExpr()); } const Expr *VisitParenExpr(const ParenExpr *E) { @@ -1060,21 +1073,25 @@ using RecIndicesTy = SmallVector<std::pair<const RecordDecl *, llvm::Value *>, 8>; static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD, - const FieldDecl *FD, RecIndicesTy &Indices) { + const FieldDecl *Field, + RecIndicesTy &Indices) { const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD); int64_t FieldNo = -1; - for (const Decl *D : RD->decls()) { - if (const auto *Field = dyn_cast<FieldDecl>(D)) { - FieldNo = Layout.getLLVMFieldNo(Field); - if (FD == Field) { - Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo))); - return true; - } + for (const FieldDecl *FD : RD->fields()) { + if (!Layout.containsFieldDecl(FD)) + // This could happen if the field has a struct type that's empty. I don't + // know why either. + continue; + + FieldNo = Layout.getLLVMFieldNo(FD); + if (FD == Field) { + Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo))); + return true; } - if (const auto *Record = dyn_cast<RecordDecl>(D)) { - ++FieldNo; - if (getGEPIndicesToField(CGF, Record, FD, Indices)) { + QualType Ty = FD->getType(); + if (Ty->isRecordType()) { + if (getGEPIndicesToField(CGF, Ty->getAsRecordDecl(), Field, Indices)) { if (RD->isUnion()) FieldNo = 0; Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo))); @@ -1104,19 +1121,15 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr( return nullptr; llvm::Value *Res = nullptr; - if (const auto *DRE = dyn_cast<DeclRefExpr>(StructBase)) { - Res = EmitDeclRefLValue(DRE).getPointer(*this); - Res = Builder.CreateAlignedLoad(ConvertType(DRE->getType()), Res, - getPointerAlign(), "dre.load"); - } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) { - LValue LV = EmitMemberExpr(ME); - Address Addr = LV.getAddress(*this); - Res = Addr.getPointer(); - } else if (StructBase->getType()->isPointerType()) { + if (StructBase->getType()->isPointerType()) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo); - Res = Addr.getPointer(); + Res = Addr.emitRawPointer(*this); + } else if (StructBase->isLValue()) { + LValue LV = EmitLValue(StructBase); + Address Addr = LV.getAddress(); + Res = Addr.emitRawPointer(*this); } else { return nullptr; } @@ -1136,38 +1149,19 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr( } const FieldDecl *CodeGenFunction::FindCountedByField(const FieldDecl *FD) { - if (!FD || !FD->hasAttr<CountedByAttr>()) - return nullptr; - - const auto *CBA = FD->getAttr<CountedByAttr>(); - if (!CBA) - return nullptr; - - auto GetNonAnonStructOrUnion = - [](const RecordDecl *RD) -> const RecordDecl * { - while (RD && RD->isAnonymousStructOrUnion()) { - const auto *R = dyn_cast<RecordDecl>(RD->getDeclContext()); - if (!R) - return nullptr; - RD = R; - } - return RD; - }; - const RecordDecl *EnclosingRD = GetNonAnonStructOrUnion(FD->getParent()); - if (!EnclosingRD) + if (!FD) return nullptr; - DeclarationName DName(CBA->getCountedByField()); - DeclContext::lookup_result Lookup = EnclosingRD->lookup(DName); - - if (Lookup.empty()) + const auto *CAT = FD->getType()->getAs<CountAttributedType>(); + if (!CAT) return nullptr; - const NamedDecl *ND = Lookup.front(); - if (const auto *IFD = dyn_cast<IndirectFieldDecl>(ND)) - ND = IFD->getAnonField(); + const auto *CountDRE = cast<DeclRefExpr>(CAT->getCountExpr()); + const auto *CountDecl = CountDRE->getDecl(); + if (const auto *IFD = dyn_cast<IndirectFieldDecl>(CountDecl)) + CountDecl = IFD->getAnonField(); - return dyn_cast<FieldDecl>(ND); + return dyn_cast<FieldDecl>(CountDecl); } void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, @@ -1301,8 +1295,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr = Address(Addr.getPointer(), Addr.getElementType(), Align, - IsKnownNonNull); + Addr.setAlignment(Align); } } @@ -1319,9 +1312,10 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, CGF.ConvertTypeForMem(E->getType()->getPointeeType()); Addr = Addr.withElementType(ElemTy); if (CE->getCastKind() == CK_AddressSpaceConversion) - Addr = CGF.Builder.CreateAddrSpaceCast(Addr, - CGF.ConvertType(E->getType())); - return Addr; + Addr = CGF.Builder.CreateAddrSpaceCast( + Addr, CGF.ConvertType(E->getType()), ElemTy); + return CGF.authPointerToPointerCast(Addr, CE->getSubExpr()->getType(), + CE->getType()); } break; @@ -1360,7 +1354,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull); if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); - return LV.getAddress(CGF); + return LV.getAddress(); } } @@ -1375,7 +1369,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull); if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); - return LV.getAddress(CGF); + return LV.getAddress(); } } } @@ -1383,10 +1377,9 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, // TODO: conditional operators, comma. // Otherwise, use the alignment of the type. - CharUnits Align = - CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType()); - return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull); + return CGF.makeNaturalAddressForPointer( + CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(), + /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull); } /// EmitPointerWithAlignment - Given an expression of pointer type, try to @@ -1487,8 +1480,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase())) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), - LV.getAlignment(), SkippedChecks); + EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks); } return LV; } @@ -1599,12 +1591,12 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E, if (LV.isSimple()) { // Defend against branches out of gnu statement expressions surrounded by // cleanups. - Address Addr = LV.getAddress(*this); - llvm::Value *V = Addr.getPointer(); + Address Addr = LV.getAddress(); + llvm::Value *V = Addr.getBasePointer(); Scope.ForceCleanup({&V}); - return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()), - LV.getType(), getContext(), LV.getBaseInfo(), - LV.getTBAAInfo()); + Addr.replaceBasePointer(V); + return LValue::MakeAddr(Addr, LV.getType(), getContext(), + LV.getBaseInfo(), LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1636,8 +1628,8 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E, return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E)); case Expr::MatrixSubscriptExprClass: return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E)); - case Expr::OMPArraySectionExprClass: - return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E)); + case Expr::ArraySectionExprClass: + return EmitArraySectionExpr(cast<ArraySectionExpr>(E)); case Expr::ExtVectorElementExprClass: return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E)); case Expr::CXXThisExprClass: @@ -1675,6 +1667,8 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E, return EmitCoawaitLValue(cast<CoawaitExpr>(E)); case Expr::CoyieldExprClass: return EmitCoyieldLValue(cast<CoyieldExpr>(E)); + case Expr::PackIndexingExprClass: + return EmitLValue(cast<PackIndexingExpr>(E)->getSelectedExpr()); } } @@ -1846,7 +1840,7 @@ llvm::Value *CodeGenFunction::emitScalarConstant( llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue, SourceLocation Loc) { - return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(), + return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(), lvalue.getType(), Loc, lvalue.getBaseInfo(), lvalue.getTBAAInfo(), lvalue.isNontemporal()); } @@ -1946,7 +1940,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isNontemporal) { - if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) + if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -1996,6 +1990,9 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal(); } + Addr = + Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType())); + llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile); if (isNontemporal) { llvm::MDNode *Node = llvm::MDNode::get( @@ -2018,27 +2015,33 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, return EmitFromMemory(Load, Ty); } +/// Converts a scalar value from its primary IR type (as returned +/// by ConvertType) to its load/store type (as returned by +/// convertTypeForLoadStore). llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) { - // Bool has a different representation in memory than in registers. - if (hasBooleanRepresentation(Ty)) { - // This should really always be an i1, but sometimes it's already - // an i8, and it's awkward to track those cases down. - if (Value->getType()->isIntegerTy(1)) - return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool"); - assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) && - "wrong value rep of bool"); + if (hasBooleanRepresentation(Ty) || Ty->isBitIntType()) { + llvm::Type *StoreTy = convertTypeForLoadStore(Ty, Value->getType()); + bool Signed = Ty->isSignedIntegerOrEnumerationType(); + return Builder.CreateIntCast(Value, StoreTy, Signed, "storedv"); + } + + if (Ty->isExtVectorBoolType()) { + llvm::Type *StoreTy = convertTypeForLoadStore(Ty, Value->getType()); + // Expand to the memory bit width. + unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits(); + // <N x i1> --> <P x i1>. + Value = emitBoolVecConversion(Value, MemNumElems, "insertvec"); + // <P x i1> --> iP. + Value = Builder.CreateBitCast(Value, StoreTy); } return Value; } +/// Converts a scalar value from its load/store type (as returned +/// by convertTypeForLoadStore) to its primary IR type (as returned +/// by ConvertType). llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { - // Bool has a different representation in memory than in registers. - if (hasBooleanRepresentation(Ty)) { - assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) && - "wrong value rep of bool"); - return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool"); - } if (Ty->isExtVectorBoolType()) { const auto *RawIntTy = Value->getType(); // Bitcast iP --> <P x i1>. @@ -2051,13 +2054,19 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { return emitBoolVecConversion(V, ValNumElems, "extractvec"); } + if (hasBooleanRepresentation(Ty) || Ty->isBitIntType()) { + llvm::Type *ResTy = ConvertType(Ty); + return Builder.CreateTrunc(Value, ResTy, "loadedv"); + } + return Value; } // Convert the pointer of \p Addr to a pointer to a vector (the value type of // MatrixType), if it points to a array (the memory type of MatrixType). -static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, - bool IsVector = true) { +static RawAddress MaybeConvertMatrixAddress(RawAddress Addr, + CodeGenFunction &CGF, + bool IsVector = true) { auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType()); if (ArrayTy && IsVector) { auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), @@ -2082,7 +2091,7 @@ static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, // (VectorType). static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue, bool isInit, CodeGenFunction &CGF) { - Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF, + Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(), CGF, value->getType()->isVectorTy()); CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit, @@ -2094,7 +2103,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isInit, bool isNontemporal) { - if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) + if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2102,17 +2111,10 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, llvm::Type *SrcTy = Value->getType(); if (const auto *ClangVecTy = Ty->getAs<VectorType>()) { auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy); - if (VecTy && ClangVecTy->isExtVectorBoolType()) { - auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType()); - // Expand to the memory bit width. - unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits(); - // <N x i1> --> <P x i1>. - Value = emitBoolVecConversion(Value, MemNumElems, "insertvec"); - // <P x i1> --> iP. - Value = Builder.CreateBitCast(Value, MemIntTy); - } else if (!CGM.getCodeGenOpts().PreserveVec3Type) { + if (!CGM.getCodeGenOpts().PreserveVec3Type) { // Handle vec3 special. - if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) { + if (VecTy && !ClangVecTy->isExtVectorBoolType() && + cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) { // Our source is a vec3, do a shuffle vector to make it a vec4. Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1}, "extractVec"); @@ -2152,7 +2154,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, return; } - EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(), + EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(), lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal()); } @@ -2162,29 +2164,44 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc, CodeGenFunction &CGF) { assert(LV.getType()->isConstantMatrixType()); - Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF); + Address Addr = MaybeConvertMatrixAddress(LV.getAddress(), CGF); LV.setAddress(Addr); return RValue::get(CGF.EmitLoadOfScalar(LV, Loc)); } +RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot, + SourceLocation Loc) { + QualType Ty = LV.getType(); + switch (getEvaluationKind(Ty)) { + case TEK_Scalar: + return EmitLoadOfLValue(LV, Loc); + case TEK_Complex: + return RValue::getComplex(EmitLoadOfComplex(LV, Loc)); + case TEK_Aggregate: + EmitAggFinalDestCopy(Ty, Slot, LV, EVK_NonRValue); + return Slot.asRValue(); + } + llvm_unreachable("bad evaluation kind"); +} + /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this /// method emits the address of the lvalue, then loads the result as an rvalue, /// returning the rvalue. RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { if (LV.isObjCWeak()) { // load of a __weak object. - Address AddrWeakObj = LV.getAddress(*this); + Address AddrWeakObj = LV.getAddress(); return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this, AddrWeakObj)); } if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { // In MRC mode, we do a load+autorelease. if (!getLangOpts().ObjCAutoRefCount) { - return RValue::get(EmitARCLoadWeak(LV.getAddress(*this))); + return RValue::get(EmitARCLoadWeak(LV.getAddress())); } // In ARC mode, we load retained and then consume the value. - llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this)); + llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress()); Object = EmitObjCConsumeObject(LV.getType(), Object); return RValue::get(Object); } @@ -2419,9 +2436,9 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, case Qualifiers::OCL_Weak: if (isInit) // Initialize and then skip the primitive store. - EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal()); + EmitARCInitWeak(Dst.getAddress(), Src.getScalarVal()); else - EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(), + EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true); return; @@ -2435,7 +2452,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (Dst.isObjCWeak() && !Dst.isNonGC()) { // load of a __weak object. - Address LvalueDst = Dst.getAddress(*this); + Address LvalueDst = Dst.getAddress(); llvm::Value *src = Src.getScalarVal(); CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst); return; @@ -2443,20 +2460,18 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (Dst.isObjCStrong() && !Dst.isNonGC()) { // load of a __strong object. - Address LvalueDst = Dst.getAddress(*this); + Address LvalueDst = Dst.getAddress(); llvm::Value *src = Src.getScalarVal(); if (Dst.isObjCIvar()) { assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); llvm::Type *ResultType = IntPtrTy; Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); - llvm::Value *RHS = dst.getPointer(); + llvm::Value *RHS = dst.emitRawPointer(*this); RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); - llvm::Value *LHS = - Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, - "sub.ptr.lhs.cast"); + llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this), + ResultType, "sub.ptr.lhs.cast"); llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); - CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, - BytesBetween); + CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween); } else if (Dst.isGlobalObjCRef()) { CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst, Dst.isThreadLocalRef()); @@ -2473,7 +2488,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, llvm::Value **Result) { const CGBitFieldInfo &Info = Dst.getBitFieldInfo(); - llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType()); + llvm::Type *ResLTy = convertTypeForLoadStore(Dst.getType()); Address Ptr = Dst.getBitFieldAddress(); // Get the source value, truncated to the width of the bit-field. @@ -2785,14 +2800,11 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo, TBAAAccessInfo *PointeeTBAAInfo) { llvm::LoadInst *Load = - Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); + Builder.CreateLoad(RefLVal.getAddress(), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); - - QualType PointeeType = RefLVal.getType()->getPointeeType(); - CharUnits Align = CGM.getNaturalTypeAlignment( - PointeeType, PointeeBaseInfo, PointeeTBAAInfo, - /* forPointeeType= */ true); - return Address(Load, ConvertTypeForMem(PointeeType), Align); + return makeNaturalAddressForPointer(Load, RefLVal.getType()->getPointeeType(), + CharUnits(), /*ForPointeeType=*/true, + PointeeBaseInfo, PointeeTBAAInfo); } LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { @@ -2809,10 +2821,9 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) { llvm::Value *Addr = Builder.CreateLoad(Ptr); - return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()), - CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo, - TBAAInfo, - /*forPointeeType=*/true)); + return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(), + CharUnits(), /*ForPointeeType=*/true, + BaseInfo, TBAAInfo); } LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, @@ -2862,22 +2873,22 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, return LV; } -static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM, - GlobalDecl GD) { +llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD, + llvm::Type *Ty) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); if (FD->hasAttr<WeakRefAttr>()) { - ConstantAddress aliasee = CGM.GetWeakRefReference(FD); + ConstantAddress aliasee = GetWeakRefReference(FD); return aliasee.getPointer(); } - llvm::Constant *V = CGM.GetAddrOfFunction(GD); + llvm::Constant *V = GetAddrOfFunction(GD, Ty); return V; } static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E, GlobalDecl GD) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); - llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD); + llvm::Constant *V = CGF.CGM.getFunctionPointer(GD); CharUnits Alignment = CGF.getContext().getDeclAlign(FD); return CGF.MakeAddrLValue(V, E->getType(), Alignment, AlignmentSource::Decl); @@ -3008,7 +3019,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { /* BaseInfo= */ nullptr, /* TBAAInfo= */ nullptr, /* forPointeeType= */ true); - Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); + Addr = makeNaturalAddressForPointer(Val, T, Alignment); } return MakeAddrLValue(Addr, T, AlignmentSource::Decl); } @@ -3039,12 +3050,13 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { LValue CapLVal = EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), CapturedStmtInfo->getContextValue()); - Address LValueAddress = CapLVal.getAddress(*this); - CapLVal = MakeAddrLValue( - Address(LValueAddress.getPointer(), LValueAddress.getElementType(), - getContext().getDeclAlign(VD)), - CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), - CapLVal.getTBAAInfo()); + Address LValueAddress = CapLVal.getAddress(); + CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this), + LValueAddress.getElementType(), + getContext().getDeclAlign(VD)), + CapLVal.getType(), + LValueBaseInfo(AlignmentSource::Decl), + CapLVal.getTBAAInfo()); // Mark lvalue as nontemporal if the variable is marked as nontemporal // in simd context. if (getLangOpts().OpenMP && @@ -3100,7 +3112,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // Handle threadlocal function locals. if (VD->getTLSKind() != VarDecl::TLS_None) addr = addr.withPointer( - Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull); + Builder.CreateThreadLocalAddress(addr.getBasePointer()), + NotKnownNonNull); // Check for OpenMP threadprivate variables. if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && @@ -3139,21 +3152,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { return LV; } - if (const auto *FD = dyn_cast<FunctionDecl>(ND)) { - LValue LV = EmitFunctionDeclLValue(*this, E, FD); - - // Emit debuginfo for the function declaration if the target wants to. - if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) { - if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) { - auto *Fn = - cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts()); - if (!Fn->getSubprogram()) - DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn); - } - } - - return LV; - } + if (const auto *FD = dyn_cast<FunctionDecl>(ND)) + return EmitFunctionDeclLValue(*this, E, FD); // FIXME: While we're emitting a binding from an enclosing scope, all other // DeclRefExprs we see should be implicitly treated as if they also refer to @@ -3227,7 +3227,7 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { // __real is valid on scalars. This is a faster way of testing that. // __imag can only produce an rvalue on scalars. if (E->getOpcode() == UO_Real && - !LV.getAddress(*this).getElementType()->isStructTy()) { + !LV.getAddress().getElementType()->isStructTy()) { assert(E->getSubExpr()->getType()->isArithmeticType()); return LV; } @@ -3236,8 +3236,8 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { Address Component = (E->getOpcode() == UO_Real - ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType()) - : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType())); + ? emitAddrOfRealComponent(LV.getAddress(), LV.getType()) + : emitAddrOfImagComponent(LV.getAddress(), LV.getType())); LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(), CGM.getTBAAInfoForSubobject(LV, T)); ElemLV.getQuals().addQualifiers(LV.getQuals()); @@ -3368,7 +3368,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { // Pointers are passed directly, everything else is passed by address. if (!V->getType()->isPointerTy()) { - Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); + RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType()); Builder.CreateStore(V, Ptr); V = Ptr.getPointer(); } @@ -3544,6 +3544,17 @@ void CodeGenFunction::EmitCheck( Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check; } + if (ClSanitizeGuardChecks) { + llvm::Value *Allow = + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::allow_ubsan_check), + llvm::ConstantInt::get(CGM.Int8Ty, CheckHandler)); + + for (llvm::Value **Cond : {&FatalCond, &RecoverableCond, &TrapCond}) { + if (*Cond) + *Cond = Builder.CreateOr(*Cond, Builder.CreateNot(Allow)); + } + } + if (TrapCond) EmitTrapCheck(TrapCond, CheckHandler); if (!FatalCond && !RecoverableCond) @@ -3570,9 +3581,8 @@ void CodeGenFunction::EmitCheck( llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName); llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers); // Give hint that we very much don't expect to execute the handler - // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp llvm::MDBuilder MDHelper(getLLVMContext()); - llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1); + llvm::MDNode *Node = MDHelper.createLikelyBranchWeights(); Branch->setMetadata(llvm::LLVMContext::MD_prof, Node); EmitBlock(Handlers); @@ -3640,7 +3650,7 @@ void CodeGenFunction::EmitCfiSlowPathCheck( llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB); llvm::MDBuilder MDHelper(getLLVMContext()); - llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1); + llvm::MDNode *Node = MDHelper.createLikelyBranchWeights(); BI->setMetadata(llvm::LLVMContext::MD_prof, Node); EmitBlock(CheckBB); @@ -3680,12 +3690,29 @@ void CodeGenFunction::EmitCfiSlowPathCheck( // symbol in LTO mode. void CodeGenFunction::EmitCfiCheckStub() { llvm::Module *M = &CGM.getModule(); - auto &Ctx = M->getContext(); + ASTContext &C = getContext(); + QualType QInt64Ty = C.getIntTypeForBitwidth(64, false); + + FunctionArgList FnArgs; + ImplicitParamDecl ArgCallsiteTypeId(C, QInt64Ty, ImplicitParamKind::Other); + ImplicitParamDecl ArgAddr(C, C.VoidPtrTy, ImplicitParamKind::Other); + ImplicitParamDecl ArgCFICheckFailData(C, C.VoidPtrTy, + ImplicitParamKind::Other); + FnArgs.push_back(&ArgCallsiteTypeId); + FnArgs.push_back(&ArgAddr); + FnArgs.push_back(&ArgCFICheckFailData); + const CGFunctionInfo &FI = + CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, FnArgs); + llvm::Function *F = llvm::Function::Create( - llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false), + llvm::FunctionType::get(VoidTy, {Int64Ty, VoidPtrTy, VoidPtrTy}, false), llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M); + CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false); + CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F); F->setAlignment(llvm::Align(4096)); CGM.setDSOLocal(F); + + llvm::LLVMContext &Ctx = M->getContext(); llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F); // CrossDSOCFI pass is not executed if there is no executable code. SmallVector<llvm::Value*> Args{F->getArg(2), F->getArg(1)}; @@ -3805,7 +3832,7 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked, // If we're optimizing, collapse all calls to trap down to just one per // check-type per function to save on code size. - if (TrapBBs.size() <= CheckHandlerID) + if ((int)TrapBBs.size() <= CheckHandlerID) TrapBBs.resize(CheckHandlerID + 1); llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID]; @@ -3826,9 +3853,10 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked, llvm::CallInst *TrapCall = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::ubsantrap), - llvm::ConstantInt::get(CGM.Int8Ty, ClSanitizeDebugDeoptimization - ? TrapBB->getParent()->size() - : CheckHandlerID)); + llvm::ConstantInt::get(CGM.Int8Ty, + ClSanitizeDebugDeoptimization + ? TrapBB->getParent()->size() + : static_cast<uint64_t>(CheckHandlerID))); if (!CGM.getCodeGenOpts().TrapFuncName.empty()) { auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name", @@ -3864,7 +3892,7 @@ Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E, // Expressions of array type can't be bitfields or vector elements. LValue LV = EmitLValue(E); - Address Addr = LV.getAddress(*this); + Address Addr = LV.getAddress(); // If the array type was an incomplete type, we need to make sure // the decay ends up being the right type. @@ -3924,6 +3952,21 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, } } +static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, + ArrayRef<llvm::Value *> indices, + llvm::Type *elementType, bool inbounds, + bool signedIndices, SourceLocation loc, + CharUnits align, + const llvm::Twine &name = "arrayidx") { + if (inbounds) { + return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices, + CodeGenFunction::NotSubtraction, loc, + align, name); + } else { + return CGF.Builder.CreateGEP(addr, indices, elementType, align, name); + } +} + static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx, CharUnits eltSize) { @@ -3971,7 +4014,7 @@ static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF, llvm::Function *Fn = CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset); - llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()}); + llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)}); return Address(Call, Addr.getElementType(), Addr.getAlignment()); } @@ -4034,7 +4077,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, // We can use that to compute the best alignment of the element. CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType); CharUnits eltAlign = - getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); if (hasBPFPreserveStaticOffset(Base)) addr = wrapWithBPFPreserveStaticOffset(CGF, addr); @@ -4043,19 +4086,19 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back()); if (!LastIndex || (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { - eltPtr = emitArraySubscriptGEP( - CGF, addr.getElementType(), addr.getPointer(), indices, inbounds, - signedIndices, loc, name); + addr = emitArraySubscriptGEP(CGF, addr, indices, + CGF.ConvertTypeForMem(eltType), inbounds, + signedIndices, loc, eltAlign, name); + return addr; } else { // Remember the original array subscript for bpf target unsigned idx = LastIndex->getZExtValue(); llvm::DIType *DbgInfo = nullptr; if (arrayType) DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc); - eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(), - addr.getPointer(), - indices.size() - 1, - idx, DbgInfo); + eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex( + addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1, + idx, DbgInfo); } return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); @@ -4147,15 +4190,14 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // If the base is a vector type, then we are forming a vector element lvalue // with this subscript. - if (E->getBase()->getType()->isVectorType() && + if (E->getBase()->getType()->isSubscriptableVectorType() && !isa<ExtVectorElementExpr>(E->getBase())) { // Emit the vector as an lvalue to get its address. LValue LHS = EmitLValue(E->getBase()); auto *Idx = EmitIdxAfterBase(/*Promote*/false); assert(LHS.isSimple() && "Can only subscript lvalue vectors here!"); - return LValue::MakeVectorElt(LHS.getAddress(*this), Idx, - E->getBase()->getType(), LHS.getBaseInfo(), - TBAAAccessInfo()); + return LValue::MakeVectorElt(LHS.getAddress(), Idx, E->getBase()->getType(), + LHS.getBaseInfo(), TBAAAccessInfo()); } // All the other cases basically behave like simple offsetting. @@ -4224,8 +4266,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, CharUnits EltAlign = getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); llvm::Value *EltPtr = - emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx, - false, SignedIndices, E->getExprLoc()); + emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this), + ScaledIdx, false, SignedIndices, E->getExprLoc()); Addr = Address(EltPtr, OrigBaseElemTy, EltAlign); } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { // If this is A[i] where A is an array, the frontend will have decayed the @@ -4257,7 +4299,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, if (const auto *ME = dyn_cast<MemberExpr>(Array); ME && ME->isFlexibleArrayMemberLike(getContext(), StrictFlexArraysLevel) && - ME->getMemberDecl()->hasAttr<CountedByAttr>()) { + ME->getMemberDecl()->getType()->isCountAttributedType()) { const FieldDecl *FAMDecl = dyn_cast<FieldDecl>(ME->getMemberDecl()); if (const FieldDecl *CountFD = FindCountedByField(FAMDecl)) { if (std::optional<int64_t> Diff = @@ -4267,11 +4309,11 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // Create a GEP with a byte offset between the FAM and count and // use that to load the count value. Addr = Builder.CreatePointerBitCastOrAddrSpaceCast( - ArrayLV.getAddress(*this), Int8PtrTy, Int8Ty); + ArrayLV.getAddress(), Int8PtrTy, Int8Ty); llvm::Type *CountTy = ConvertType(CountFD->getType()); llvm::Value *Res = Builder.CreateInBoundsGEP( - Int8Ty, Addr.getPointer(), + Int8Ty, Addr.emitRawPointer(*this), Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep"); Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(), ".counted_by.load"); @@ -4287,7 +4329,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // Propagate the alignment from the array itself to the result. QualType arrayType = Array->getType(); Addr = emitArraySubscriptGEP( - *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx}, + *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx}, E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices, E->getExprLoc(), &arrayType, E->getBase()); EltBaseInfo = ArrayLV.getBaseInfo(); @@ -4326,7 +4368,7 @@ LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) { llvm::Value *FinalIdx = Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx); return LValue::MakeMatrixElt( - MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx, + MaybeConvertMatrixAddress(Base.getAddress(), *this), FinalIdx, E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo()); } @@ -4336,10 +4378,10 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, QualType BaseTy, QualType ElTy, bool IsLowerBound) { LValue BaseLVal; - if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) { - BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound); + if (auto *ASE = dyn_cast<ArraySectionExpr>(Base->IgnoreParenImpCasts())) { + BaseLVal = CGF.EmitArraySectionExpr(ASE, IsLowerBound); if (BaseTy->isArrayType()) { - Address Addr = BaseLVal.getAddress(CGF); + Address Addr = BaseLVal.getAddress(); BaseInfo = BaseLVal.getBaseInfo(); // If the array type was an incomplete type, we need to make sure @@ -4363,15 +4405,19 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo); BaseInfo.mergeForCast(TypeBaseInfo); TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo); - return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)), + return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress()), CGF.ConvertTypeForMem(ElTy), Align); } return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); } -LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, - bool IsLowerBound) { - QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(E->getBase()); +LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E, + bool IsLowerBound) { + + assert(!E->isOpenACCArraySection() && + "OpenACC Array section codegen not implemented"); + + QualType BaseTy = ArraySectionExpr::getBaseOriginalType(E->getBase()); QualType ResultExprTy; if (auto *AT = getContext().getAsArrayType(BaseTy)) ResultExprTy = AT->getElementType(); @@ -4511,15 +4557,15 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, // Propagate the alignment from the array itself to the result. EltPtr = emitArraySubscriptGEP( - *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx}, + *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx}, ResultExprTy, !getLangOpts().isSignedOverflowDefined(), /*signedIndices=*/false, E->getExprLoc()); BaseInfo = ArrayLV.getBaseInfo(); TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy); } else { - Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, - TBAAInfo, BaseTy, ResultExprTy, - IsLowerBound); + Address Base = + emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy, + ResultExprTy, IsLowerBound); EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy, !getLangOpts().isSignedOverflowDefined(), /*signedIndices=*/false, E->getExprLoc()); @@ -4571,7 +4617,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { if (Base.isSimple()) { llvm::Constant *CV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); - return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type, + return LValue::MakeExtVectorElt(Base.getAddress(), CV, type, Base.getBaseInfo(), TBAAAccessInfo()); } assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!"); @@ -4606,7 +4652,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { SkippedChecks.set(SanitizerKind::Alignment, true); if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr)) SkippedChecks.set(SanitizerKind::Null, true); - EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy, + EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy, /*Alignment=*/CharUnits::Zero(), SkippedChecks); BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); } else @@ -4640,7 +4686,8 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field, llvm::Value *ThisValue) { bool HasExplicitObjectParameter = false; - if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(CurCodeDecl)) { + const auto *MD = dyn_cast_if_present<CXXMethodDecl>(CurCodeDecl); + if (MD) { HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction(); assert(MD->getParent()->isLambda()); assert(MD->getParent() == Field->getParent()); @@ -4655,8 +4702,19 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field, LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(), AlignmentSource::Decl); else - LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(), - D->getType().getNonReferenceType()); + LambdaLV = MakeAddrLValue(AddrOfExplicitObject, + D->getType().getNonReferenceType()); + + // Make sure we have an lvalue to the lambda itself and not a derived class. + auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl(); + auto *LambdaTy = cast<CXXRecordDecl>(Field->getParent()); + if (ThisTy != LambdaTy) { + const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(MD); + Address Base = GetAddressOfBaseClass( + LambdaLV.getAddress(), ThisTy, BasePathArray.begin(), + BasePathArray.end(), /*NullCheckValue=*/false, SourceLocation()); + LambdaLV = MakeAddrLValue(Base, QualType{LambdaTy->getTypeForDecl(), 0}); + } } else { QualType LambdaTagType = getContext().getTagDeclType(Field->getParent()); LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType); @@ -4677,7 +4735,7 @@ unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec, for (auto *F : Rec->getDefinition()->fields()) { if (I == FieldIndex) break; - if (F->isUnnamedBitfield()) + if (F->isUnnamedBitField()) Skipped++; I++; } @@ -4703,7 +4761,7 @@ static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base, /// The resulting address doesn't necessarily have the right type. static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base, const FieldDecl *field) { - if (field->isZeroSize(CGF.getContext())) + if (isEmptyFieldForLayout(CGF.getContext(), field)) return emitAddrOfZeroSizeField(CGF, base, field); const RecordDecl *rec = field->getParent(); @@ -4760,7 +4818,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, field->getType() .withCVRQualifiers(base.getVRQualifiers()) .isVolatileQualified(); - Address Addr = base.getAddress(*this); + Address Addr = base.getAddress(); unsigned Idx = RL.getLLVMFieldNo(field); const RecordDecl *rec = field->getParent(); if (hasBPFPreserveStaticOffset(rec)) @@ -4836,7 +4894,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, getContext().getTypeSizeInChars(FieldType).getQuantity(); } - Address addr = base.getAddress(*this); + Address addr = base.getAddress(); if (hasBPFPreserveStaticOffset(rec)) addr = wrapWithBPFPreserveStaticOffset(*this, addr); if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) { @@ -4846,7 +4904,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // information provided by invariant.group. This is because accessing // fields may leak the real address of dynamic object, which could result // in miscompilation when leaked pointer would be compared. - auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer()); + auto *stripped = + Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this)); addr = Address(stripped, addr.getElementType(), addr.getAlignment()); } } @@ -4865,10 +4924,11 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // Remember the original union field index llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(), rec->getLocation()); - addr = Address( - Builder.CreatePreserveUnionAccessIndex( - addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), - addr.getElementType(), addr.getAlignment()); + addr = + Address(Builder.CreatePreserveUnionAccessIndex( + addr.emitRawPointer(*this), + getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), + addr.getElementType(), addr.getAlignment()); } if (FieldType->isReferenceType()) @@ -4921,7 +4981,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, if (!FieldType->isReferenceType()) return EmitLValueForField(Base, Field); - Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field); + Address V = emitAddrOfFieldStorage(*this, Base.getAddress(), Field); // Make sure that the address is pointing to the right type. llvm::Type *llvmType = ConvertTypeForMem(FieldType); @@ -5103,13 +5163,11 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue( return EmitUnsupportedLValue(expr, "conditional operator"); if (Info.LHS && Info.RHS) { - Address lhsAddr = Info.LHS->getAddress(*this); - Address rhsAddr = Info.RHS->getAddress(*this); - llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); - phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock); - phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock); - Address result(phi, lhsAddr.getElementType(), - std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); + Address lhsAddr = Info.LHS->getAddress(); + Address rhsAddr = Info.RHS->getAddress(); + Address result = mergeAddressesInConditionalExpr( + lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock, + Builder.GetInsertBlock(), expr->getType()); AlignmentSource alignSource = std::max(Info.LHS->getBaseInfo().getAlignmentSource(), Info.RHS->getBaseInfo().getAlignmentSource()); @@ -5178,6 +5236,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_FixedPointToIntegral: case CK_IntegralToFixedPoint: case CK_MatrixCast: + case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: return EmitUnsupportedLValue(E, "unexpected cast lvalue"); case CK_Dependent: @@ -5193,9 +5253,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_Dynamic: { LValue LV = EmitLValue(E->getSubExpr()); - Address V = LV.getAddress(*this); + Address V = LV.getAddress(); const auto *DCE = cast<CXXDynamicCastExpr>(E); - return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); + return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType()); } case CK_ConstructorConversion: @@ -5214,7 +5274,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { if (E->changesVolatileQualification()) LV.getQuals() = E->getType().getQualifiers(); if (LV.isSimple()) { - Address V = LV.getAddress(*this); + Address V = LV.getAddress(); if (V.isValid()) { llvm::Type *T = ConvertTypeForMem(E->getType()); if (V.getElementType() != T) @@ -5231,7 +5291,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl()); LValue LV = EmitLValue(E->getSubExpr()); - Address This = LV.getAddress(*this); + Address This = LV.getAddress(); // Perform the derived-to-base conversion Address Base = GetAddressOfBaseClass( @@ -5254,14 +5314,14 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { // Perform the base-to-derived conversion Address Derived = GetAddressOfDerivedClass( - LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(), + LV.getAddress(), DerivedClassDecl, E->path_begin(), E->path_end(), /*NullCheckValue=*/false); // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is // performed and the object is not of the derived type. if (sanitizePerformTypeCheck()) - EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), - Derived.getPointer(), E->getType()); + EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived, + E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) EmitVTablePtrCheckForCast(E->getType(), Derived, @@ -5277,7 +5337,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { CGM.EmitExplicitCastExprType(CE, this); LValue LV = EmitLValue(E->getSubExpr()); - Address V = LV.getAddress(*this).withElementType( + Address V = LV.getAddress().withElementType( ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType())); if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) @@ -5296,12 +5356,12 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { E->getSubExpr()->getType().getAddressSpace(), E->getType().getAddressSpace(), ConvertType(DestTy)); return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()), - LV.getAddress(*this).getAlignment()), + LV.getAddress().getAlignment()), E->getType(), LV.getBaseInfo(), LV.getTBAAInfo()); } case CK_ObjCObjectLValueCast: { LValue LV = EmitLValue(E->getSubExpr()); - Address V = LV.getAddress(*this).withElementType(ConvertType(E->getType())); + Address V = LV.getAddress().withElementType(ConvertType(E->getType())); return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(), CGM.getTBAAInfoForSubobject(LV, E->getType())); } @@ -5361,7 +5421,7 @@ RValue CodeGenFunction::EmitRValueForField(LValue LV, case TEK_Complex: return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc)); case TEK_Aggregate: - return FieldLV.asAggregateRValue(*this); + return FieldLV.asAggregateRValue(); case TEK_Scalar: // This routine is used to load fields one-by-one to perform a copy, so // don't load reference fields. @@ -5451,7 +5511,7 @@ static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) { // name to make it clear it's not the actual builtin. if (CGF.CurFn->getName() != FDInlineName && OnlyHasInlineBuiltinDeclaration(FD)) { - llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); + llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD); llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr); llvm::Module *M = Fn->getParent(); llvm::Function *Clone = M->getFunction(FDInlineName); @@ -5474,7 +5534,7 @@ static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) { return CGCallee::forBuiltin(builtinID, FD); } - llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); + llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD); if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice && FD->hasAttr<CUDAGlobalAttr>()) CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub( @@ -5531,7 +5591,8 @@ CGCallee CodeGenFunction::EmitCallee(const Expr *E) { GD = GlobalDecl(VD); CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD); - CGCallee callee(calleeInfo, calleePtr); + CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(functionType); + CGCallee callee(calleeInfo, calleePtr, pointerAuth); return callee; } @@ -5568,11 +5629,44 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) { break; } - RValue RV = EmitAnyExpr(E->getRHS()); + // TODO: Can we de-duplicate this code with the corresponding code in + // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works? + RValue RV; + llvm::Value *Previous = nullptr; + QualType SrcType = E->getRHS()->getType(); + // Check if LHS is a bitfield, if RHS contains an implicit cast expression + // we want to extract that value and potentially (if the bitfield sanitizer + // is enabled) use it to check for an implicit conversion. + if (E->getLHS()->refersToBitField()) { + llvm::Value *RHS = + EmitWithOriginalRHSBitfieldAssignment(E, &Previous, &SrcType); + RV = RValue::get(RHS); + } else + RV = EmitAnyExpr(E->getRHS()); + LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store); + if (RV.isScalar()) EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc()); - EmitStoreThroughLValue(RV, LV); + + if (LV.isBitField()) { + llvm::Value *Result = nullptr; + // If bitfield sanitizers are enabled we want to use the result + // to check whether a truncation or sign change has occurred. + if (SanOpts.has(SanitizerKind::ImplicitBitfieldConversion)) + EmitStoreThroughBitfieldLValue(RV, LV, &Result); + else + EmitStoreThroughBitfieldLValue(RV, LV); + + // If the expression contained an implicit conversion, make sure + // to use the value before the scalar conversion. + llvm::Value *Src = Previous ? Previous : RV.getScalarVal(); + QualType DstType = E->getLHS()->getType(); + EmitBitfieldConversionCheck(Src, SrcType, Result, DstType, + LV.getBitFieldInfo(), E->getExprLoc()); + } else + EmitStoreThroughLValue(RV, LV); + if (getLangOpts().OpenMP) CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this, E->getLHS()); @@ -5617,7 +5711,7 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) { LValue CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { - return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType()); + return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType()); } Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { @@ -5746,6 +5840,15 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true); llvm::Value *CalleePtr = Callee.getFunctionPointer(); + if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) { + // Use raw pointer since we are using the callee pointer as data here. + Address Addr = + Address(CalleePtr, CalleePtr->getType(), + CharUnits::fromQuantity( + CalleePtr->getPointerAlignment(CGM.getDataLayout())), + Callee.getPointerAuthInfo(), nullptr); + CalleePtr = Addr.emitRawPointer(*this); + } // On 32-bit Arm, the low bit of a function pointer indicates whether // it's using the Arm or Thumb instruction set. The actual first @@ -5950,7 +6053,7 @@ EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) { if (E->getOpcode() == BO_PtrMemI) { BaseAddr = EmitPointerWithAlignment(E->getLHS()); } else { - BaseAddr = EmitLValue(E->getLHS()).getAddress(*this); + BaseAddr = EmitLValue(E->getLHS()).getAddress(); } llvm::Value *OffsetV = EmitScalarExpr(E->getRHS()); @@ -5975,7 +6078,7 @@ RValue CodeGenFunction::convertTempToRValue(Address addr, case TEK_Complex: return RValue::getComplex(EmitLoadOfComplex(lvalue, loc)); case TEK_Aggregate: - return lvalue.asAggregateRValue(*this); + return lvalue.asAggregateRValue(); case TEK_Scalar: return RValue::get(EmitLoadOfScalar(lvalue, loc)); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprAgg.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprAgg.cpp index 810b28f25fa1..d9f44f4be617 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprAgg.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprAgg.cpp @@ -15,6 +15,7 @@ #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "ConstantEmitter.h" +#include "EHScopeStack.h" #include "TargetInfo.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" @@ -24,6 +25,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" using namespace clang; @@ -33,6 +35,10 @@ using namespace CodeGen; // Aggregate Expression Emitter //===----------------------------------------------------------------------===// +namespace llvm { +extern cl::opt<bool> EnableSingleByteCoverage; +} // namespace llvm + namespace { class AggExprEmitter : public StmtVisitor<AggExprEmitter> { CodeGenFunction &CGF; @@ -72,15 +78,11 @@ public: /// then loads the result into DestPtr. void EmitAggLoadOfLValue(const Expr *E); - enum ExprValueKind { - EVK_RValue, - EVK_NonRValue - }; - /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. /// SrcIsRValue is true if source comes from an RValue. void EmitFinalDestCopy(QualType type, const LValue &src, - ExprValueKind SrcValueKind = EVK_NonRValue); + CodeGenFunction::ExprValueKind SrcValueKind = + CodeGenFunction::EVK_NonRValue); void EmitFinalDestCopy(QualType type, RValue src); void EmitCopy(QualType type, const AggValueSlot &dest, const AggValueSlot &src); @@ -129,15 +131,12 @@ public: EnsureDest(E->getType()); if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) { - Address StoreDest = Dest.getAddress(); - // The emitted value is guaranteed to have the same size as the - // destination but can have a different type. Just do a bitcast in this - // case to avoid incorrect GEPs. - if (Result->getType() != StoreDest.getType()) - StoreDest = StoreDest.withElementType(Result->getType()); - - CGF.EmitAggregateStore(Result, StoreDest, - E->getType().isVolatileQualified()); + CGF.CreateCoercedStore( + Result, Dest.getAddress(), + llvm::TypeSize::getFixed( + Dest.getPreferredSize(CGF.getContext(), E->getType()) + .getQuantity()), + E->getType().isVolatileQualified()); return; } return Visit(E->getSubExpr()); @@ -235,6 +234,9 @@ public: RValue Res = CGF.EmitAtomicExpr(E); EmitFinalDestCopy(E->getType(), Res); } + void VisitPackIndexingExpr(PackIndexingExpr *E) { + Visit(E->getSelectedExpr()); + } }; } // end anonymous namespace. @@ -287,10 +289,10 @@ void AggExprEmitter::withReturnValueSlot( // Otherwise, EmitCall will emit its own, notice that it's "unused", and end // its lifetime before we have the chance to emit a proper destructor call. bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() || - (RequiresDestruction && !Dest.getAddress().isValid()); + (RequiresDestruction && Dest.isIgnored()); Address RetAddr = Address::invalid(); - Address RetAllocaAddr = Address::invalid(); + RawAddress RetAllocaAddr = RawAddress::invalid(); EHScopeStack::stable_iterator LifetimeEndBlock; llvm::Value *LifetimeSizePtr = nullptr; @@ -322,7 +324,8 @@ void AggExprEmitter::withReturnValueSlot( if (!UseTemp) return; - assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer()); + assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) != + Src.getAggregatePointer(E->getType(), CGF)); EmitFinalDestCopy(E->getType(), Src); if (!RequiresDestruction && LifetimeStartInst) { @@ -338,12 +341,13 @@ void AggExprEmitter::withReturnValueSlot( void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) { assert(src.isAggregate() && "value must be aggregate value!"); LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type); - EmitFinalDestCopy(type, srcLV, EVK_RValue); + EmitFinalDestCopy(type, srcLV, CodeGenFunction::EVK_RValue); } /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. -void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src, - ExprValueKind SrcValueKind) { +void AggExprEmitter::EmitFinalDestCopy( + QualType type, const LValue &src, + CodeGenFunction::ExprValueKind SrcValueKind) { // If Dest is ignored, then we're evaluating an aggregate expression // in a context that doesn't care about the result. Note that loads // from volatile l-values force the existence of a non-ignored @@ -355,7 +359,7 @@ void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src, LValue DstLV = CGF.MakeAddrLValue( Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type); - if (SrcValueKind == EVK_RValue) { + if (SrcValueKind == CodeGenFunction::EVK_RValue) { if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) { if (Dest.isPotentiallyAliased()) CGF.callCStructMoveAssignmentOperator(DstLV, src); @@ -374,8 +378,8 @@ void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src, } AggValueSlot srcAgg = AggValueSlot::forLValue( - src, CGF, AggValueSlot::IsDestructed, needsGC(type), - AggValueSlot::IsAliased, AggValueSlot::MayOverlap); + src, AggValueSlot::IsDestructed, needsGC(type), AggValueSlot::IsAliased, + AggValueSlot::MayOverlap); EmitCopy(type, Dest, srcAgg); } @@ -413,60 +417,51 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { ASTContext &Ctx = CGF.getContext(); LValue Array = CGF.EmitLValue(E->getSubExpr()); assert(Array.isSimple() && "initializer_list array not a simple lvalue"); - Address ArrayPtr = Array.getAddress(CGF); + Address ArrayPtr = Array.getAddress(); const ConstantArrayType *ArrayType = Ctx.getAsConstantArrayType(E->getSubExpr()->getType()); assert(ArrayType && "std::initializer_list constructed from non-array"); - // FIXME: Perform the checks on the field types in SemaInit. RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl(); RecordDecl::field_iterator Field = Record->field_begin(); - if (Field == Record->field_end()) { - CGF.ErrorUnsupported(E, "weird std::initializer_list"); - return; - } + assert(Field != Record->field_end() && + Ctx.hasSameType(Field->getType()->getPointeeType(), + ArrayType->getElementType()) && + "Expected std::initializer_list first field to be const E *"); // Start pointer. - if (!Field->getType()->isPointerType() || - !Ctx.hasSameType(Field->getType()->getPointeeType(), - ArrayType->getElementType())) { - CGF.ErrorUnsupported(E, "weird std::initializer_list"); - return; - } - AggValueSlot Dest = EnsureSlot(E->getType()); LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field); - llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); - llvm::Value *IdxStart[] = { Zero, Zero }; - llvm::Value *ArrayStart = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart"); + llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF); CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); ++Field; - - if (Field == Record->field_end()) { - CGF.ErrorUnsupported(E, "weird std::initializer_list"); - return; - } + assert(Field != Record->field_end() && + "Expected std::initializer_list to have two fields"); llvm::Value *Size = Builder.getInt(ArrayType->getSize()); LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field); - if (Field->getType()->isPointerType() && - Ctx.hasSameType(Field->getType()->getPointeeType(), - ArrayType->getElementType())) { + if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { + // Length. + CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength); + + } else { // End pointer. + assert(Field->getType()->isPointerType() && + Ctx.hasSameType(Field->getType()->getPointeeType(), + ArrayType->getElementType()) && + "Expected std::initializer_list second field to be const E *"); + llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); llvm::Value *IdxEnd[] = { Zero, Size }; llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend"); + ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd, + "arrayend"); CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); - } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { - // Length. - CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength); - } else { - CGF.ErrorUnsupported(E, "weird std::initializer_list"); - return; } + + assert(++Field == Record->field_end() && + "Expected std::initializer_list to only have two fields"); } /// Determine if E is a trivial array filler, that is, one that is @@ -500,19 +495,20 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, uint64_t NumInitElements = Args.size(); uint64_t NumArrayElements = AType->getNumElements(); + for (const auto *Init : Args) { + if (const auto *Embed = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) { + NumInitElements += Embed->getDataElementCount() - 1; + if (NumInitElements > NumArrayElements) { + NumInitElements = NumArrayElements; + break; + } + } + } + assert(NumInitElements <= NumArrayElements); QualType elementType = CGF.getContext().getAsArrayType(ArrayQTy)->getElementType(); - - // DestPtr is an array*. Construct an elementType* by drilling - // down a level. - llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); - llvm::Value *indices[] = { zero, zero }; - llvm::Value *begin = Builder.CreateInBoundsGEP( - DestPtr.getElementType(), DestPtr.getPointer(), indices, - "arrayinit.begin"); - CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); CharUnits elementAlign = DestPtr.getAlignment().alignmentOfArrayElement(elementSize); @@ -525,9 +521,12 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, elementType.isTriviallyCopyableType(CGF.getContext())) { CodeGen::CodeGenModule &CGM = CGF.CGM; ConstantEmitter Emitter(CGF); - LangAS AS = ArrayQTy.getAddressSpace(); + QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType( + CGM.getContext().removeAddrSpaceQualType(ArrayQTy), + CGM.GetGlobalConstantAddressSpace()); + LangAS AS = GVArrayQTy.getAddressSpace(); if (llvm::Constant *C = - Emitter.tryEmitForInitializer(ExprToVisit, AS, ArrayQTy)) { + Emitter.tryEmitForInitializer(ExprToVisit, AS, GVArrayQTy)) { auto GV = new llvm::GlobalVariable( CGM.getModule(), C->getType(), /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C, @@ -535,10 +534,10 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal, CGM.getContext().getTargetAddressSpace(AS)); Emitter.finalize(GV); - CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy); + CharUnits Align = CGM.getContext().getTypeAlignInChars(GVArrayQTy); GV->setAlignment(Align.getAsAlign()); Address GVAddr(GV, GV->getValueType(), Align); - EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, ArrayQTy)); + EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, GVArrayQTy)); return; } } @@ -548,51 +547,63 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // For that, we'll need an EH cleanup. QualType::DestructionKind dtorKind = elementType.isDestructedType(); Address endOfInit = Address::invalid(); - EHScopeStack::stable_iterator cleanup; - llvm::Instruction *cleanupDominator = nullptr; - if (CGF.needsEHCleanup(dtorKind)) { + CodeGenFunction::CleanupDeactivationScope deactivation(CGF); + + llvm::Value *begin = DestPtr.emitRawPointer(CGF); + if (dtorKind) { + CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF); // In principle we could tell the cleanup where we are more // directly, but the control flow can get so varied here that it // would actually be quite complex. Therefore we go through an // alloca. + llvm::Instruction *dominatingIP = + Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(CGF.Int8PtrTy)); endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(), "arrayinit.endOfInit"); - cleanupDominator = Builder.CreateStore(begin, endOfInit); + Builder.CreateStore(begin, endOfInit); CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType, elementAlign, CGF.getDestroyer(dtorKind)); - cleanup = CGF.EHStack.stable_begin(); + cast<EHCleanupScope>(*CGF.EHStack.find(CGF.EHStack.stable_begin())) + .AddAuxAllocas(allocaTracker.Take()); - // Otherwise, remember that we didn't need a cleanup. - } else { - dtorKind = QualType::DK_none; + CGF.DeferredDeactivationCleanupStack.push_back( + {CGF.EHStack.stable_begin(), dominatingIP}); } llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1); - // The 'current element to initialize'. The invariants on this - // variable are complicated. Essentially, after each iteration of - // the loop, it points to the last initialized element, except - // that it points to the beginning of the array before any - // elements have been initialized. - llvm::Value *element = begin; - - // Emit the explicit initializers. - for (uint64_t i = 0; i != NumInitElements; ++i) { - // Advance to the next element. - if (i > 0) { + auto Emit = [&](Expr *Init, uint64_t ArrayIndex) { + llvm::Value *element = begin; + if (ArrayIndex > 0) { element = Builder.CreateInBoundsGEP( - llvmElementType, element, one, "arrayinit.element"); + llvmElementType, begin, + llvm::ConstantInt::get(CGF.SizeTy, ArrayIndex), "arrayinit.element"); // Tell the cleanup that it needs to destroy up to this // element. TODO: some of these stores can be trivially // observed to be unnecessary. - if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); + if (endOfInit.isValid()) + Builder.CreateStore(element, endOfInit); } LValue elementLV = CGF.MakeAddrLValue( Address(element, llvmElementType, elementAlign), elementType); - EmitInitializationToLValue(Args[i], elementLV); + EmitInitializationToLValue(Init, elementLV); + return true; + }; + + unsigned ArrayIndex = 0; + // Emit the explicit initializers. + for (uint64_t i = 0; i != NumInitElements; ++i) { + if (ArrayIndex >= NumInitElements) + break; + if (auto *EmbedS = dyn_cast<EmbedExpr>(Args[i]->IgnoreParenImpCasts())) { + EmbedS->doForEachDataElement(Emit, ArrayIndex); + } else { + Emit(Args[i], ArrayIndex); + ArrayIndex++; + } } // Check whether there's a non-trivial array-fill expression. @@ -609,9 +620,12 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // do { *array++ = filler; } while (array != end); // Advance to the start of the rest of the array. + llvm::Value *element = begin; if (NumInitElements) { element = Builder.CreateInBoundsGEP( - llvmElementType, element, one, "arrayinit.start"); + llvmElementType, element, + llvm::ConstantInt::get(CGF.SizeTy, NumInitElements), + "arrayinit.start"); if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); } @@ -661,9 +675,6 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, CGF.EmitBlock(endBB); } - - // Leave the partial-array cleanup if we entered one. - if (dtorKind) CGF.DeactivateCleanupBlock(cleanup, cleanupDominator); } //===----------------------------------------------------------------------===// @@ -732,7 +743,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { CodeGenFunction::TCK_Load); // FIXME: Do we also need to handle property references here? if (LV.isSimple()) - CGF.EmitDynamicCast(LV.getAddress(CGF), cast<CXXDynamicCastExpr>(E)); + CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E)); else CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast"); @@ -765,8 +776,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { } LValue SourceLV = CGF.EmitLValue(E->getSubExpr()); - Address SourceAddress = - SourceLV.getAddress(CGF).withElementType(CGF.Int8Ty); + Address SourceAddress = SourceLV.getAddress().withElementType(CGF.Int8Ty); Address DestAddress = Dest.getAddress().withElementType(CGF.Int8Ty); llvm::Value *SizeVal = llvm::ConstantInt::get( CGF.SizeTy, @@ -873,6 +883,9 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { [[fallthrough]]; + case CK_HLSLArrayRValue: + Visit(E->getSubExpr()); + break; case CK_NoOp: case CK_UserDefinedConversion: @@ -930,6 +943,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { case CK_BuiltinFnToFnPtr: case CK_ZeroToOCLOpaqueType: case CK_MatrixCast: + case CK_HLSLVectorTruncation: case CK_IntToOCLSampler: case CK_FloatingToFixedPoint: @@ -1051,7 +1065,7 @@ void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) { if (RV.isScalar()) return {RV.getScalarVal(), nullptr}; if (RV.isAggregate()) - return {RV.getAggregatePointer(), nullptr}; + return {RV.getAggregatePointer(E->getType(), CGF), nullptr}; assert(RV.isComplex()); return RV.getComplexVal(); }; @@ -1212,7 +1226,7 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { } EmitCopy(E->getLHS()->getType(), - AggValueSlot::forLValue(LHS, CGF, AggValueSlot::IsDestructed, + AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()), AggValueSlot::IsAliased, AggValueSlot::MayOverlap), @@ -1234,7 +1248,7 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { // Codegen the RHS so that it stores directly into the LHS. AggValueSlot LHSSlot = AggValueSlot::forLValue( - LHS, CGF, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()), + LHS, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()), AggValueSlot::IsAliased, AggValueSlot::MayOverlap); // A non-volatile aggregate destination might have volatile member. if (!LHSSlot.isVolatile() && @@ -1275,7 +1289,10 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { eval.begin(CGF); CGF.EmitBlock(LHSBlock); - CGF.incrementProfileCounter(E); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E->getTrueExpr()); + else + CGF.incrementProfileCounter(E); Visit(E->getTrueExpr()); eval.end(CGF); @@ -1290,6 +1307,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { eval.begin(CGF); CGF.EmitBlock(RHSBlock); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E->getFalseExpr()); Visit(E->getFalseExpr()); eval.end(CGF); @@ -1298,6 +1317,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { E->getType()); CGF.EmitBlock(ContBlock); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E); } void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) { @@ -1306,15 +1327,13 @@ void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) { void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) { Address ArgValue = Address::invalid(); - Address ArgPtr = CGF.EmitVAArg(VE, ArgValue); + CGF.EmitVAArg(VE, ArgValue, Dest); // If EmitVAArg fails, emit an error. - if (!ArgPtr.isValid()) { + if (!ArgValue.isValid()) { CGF.ErrorUnsupported(VE, "aggregate va_arg expression"); return; } - - EmitFinalDestCopy(VE->getType(), CGF.MakeAddrLValue(ArgPtr, VE->getType())); } void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) { @@ -1353,9 +1372,8 @@ AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) { LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType()); // We'll need to enter cleanup scopes in case any of the element - // initializers throws an exception. - SmallVector<EHScopeStack::stable_iterator, 16> Cleanups; - llvm::Instruction *CleanupDominator = nullptr; + // initializers throws an exception or contains branch out of the expressions. + CodeGenFunction::CleanupDeactivationScope scope(CGF); CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin(); for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(), @@ -1374,28 +1392,12 @@ AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) { if (QualType::DestructionKind DtorKind = CurField->getType().isDestructedType()) { assert(LV.isSimple()); - if (CGF.needsEHCleanup(DtorKind)) { - if (!CleanupDominator) - CleanupDominator = CGF.Builder.CreateAlignedLoad( - CGF.Int8Ty, - llvm::Constant::getNullValue(CGF.Int8PtrTy), - CharUnits::One()); // placeholder - - CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), CurField->getType(), - CGF.getDestroyer(DtorKind), false); - Cleanups.push_back(CGF.EHStack.stable_begin()); - } + if (DtorKind) + CGF.pushDestroyAndDeferDeactivation(NormalAndEHCleanup, LV.getAddress(), + CurField->getType(), + CGF.getDestroyer(DtorKind), false); } } - - // Deactivate all the partial cleanups in reverse order, which - // generally means popping them. - for (unsigned i = Cleanups.size(); i != 0; --i) - CGF.DeactivateCleanupBlock(Cleanups[i-1], CleanupDominator); - - // Destroy the placeholder if we made one. - if (CleanupDominator) - CleanupDominator->eraseFromParent(); } void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) { @@ -1454,6 +1456,7 @@ static bool castPreservesZero(const CastExpr *CE) { case CK_MatrixCast: case CK_NonAtomicToAtomic: case CK_AtomicToNonAtomic: + case CK_HLSLVectorTruncation: return true; case CK_BaseToDerivedMemberPointer: @@ -1505,6 +1508,7 @@ static bool castPreservesZero(const CastExpr *CE) { case CK_LValueToRValue: case CK_LValueToRValueBitCast: case CK_UncheckedDerivedToBase: + case CK_HLSLArrayRValue: return false; } llvm_unreachable("Unhandled clang::CastKind enum"); @@ -1569,7 +1573,7 @@ AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) { return; case TEK_Aggregate: CGF.EmitAggExpr( - E, AggValueSlot::forLValue(LV, CGF, AggValueSlot::IsDestructed, + E, AggValueSlot::forLValue(LV, AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, AggValueSlot::MayOverlap, Dest.isZeroed())); @@ -1608,7 +1612,7 @@ void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) { // There's a potential optimization opportunity in combining // memsets; that would be easy for arrays, but relatively // difficult for structures with the current code. - CGF.EmitNullInitialization(lv.getAddress(CGF), lv.getType()); + CGF.EmitNullInitialization(lv.getAddress(), lv.getType()); } } @@ -1682,14 +1686,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( // We'll need to enter cleanup scopes in case any of the element // initializers throws an exception. SmallVector<EHScopeStack::stable_iterator, 16> cleanups; - llvm::Instruction *cleanupDominator = nullptr; - auto addCleanup = [&](const EHScopeStack::stable_iterator &cleanup) { - cleanups.push_back(cleanup); - if (!cleanupDominator) // create placeholder once needed - cleanupDominator = CGF.Builder.CreateAlignedLoad( - CGF.Int8Ty, llvm::Constant::getNullValue(CGF.Int8PtrTy), - CharUnits::One()); - }; + CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF); unsigned curInitIndex = 0; @@ -1712,10 +1709,8 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( CGF.EmitAggExpr(InitExprs[curInitIndex++], AggSlot); if (QualType::DestructionKind dtorKind = - Base.getType().isDestructedType()) { - CGF.pushDestroy(dtorKind, V, Base.getType()); - addCleanup(CGF.EHStack.stable_begin()); - } + Base.getType().isDestructedType()) + CGF.pushDestroyAndDeferDeactivation(dtorKind, V, Base.getType()); } } @@ -1732,7 +1727,9 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( // Make sure that it's really an empty and not a failure of // semantic analysis. for (const auto *Field : record->fields()) - assert((Field->isUnnamedBitfield() || Field->isAnonymousStructOrUnion()) && "Only unnamed bitfields or ananymous class allowed"); + assert( + (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) && + "Only unnamed bitfields or anonymous class allowed"); #endif return; } @@ -1760,7 +1757,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( break; // Always skip anonymous bitfields. - if (field->isUnnamedBitfield()) + if (field->isUnnamedBitField()) continue; // We're done if we reach the end of the explicit initializers, we @@ -1786,37 +1783,16 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( // Push a destructor if necessary. // FIXME: if we have an array of structures, all explicitly // initialized, we can end up pushing a linear number of cleanups. - bool pushedCleanup = false; if (QualType::DestructionKind dtorKind = field->getType().isDestructedType()) { assert(LV.isSimple()); - if (CGF.needsEHCleanup(dtorKind)) { - CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), field->getType(), - CGF.getDestroyer(dtorKind), false); - addCleanup(CGF.EHStack.stable_begin()); - pushedCleanup = true; + if (dtorKind) { + CGF.pushDestroyAndDeferDeactivation(NormalAndEHCleanup, LV.getAddress(), + field->getType(), + CGF.getDestroyer(dtorKind), false); } } - - // If the GEP didn't get used because of a dead zero init or something - // else, clean it up for -O0 builds and general tidiness. - if (!pushedCleanup && LV.isSimple()) - if (llvm::GetElementPtrInst *GEP = - dyn_cast<llvm::GetElementPtrInst>(LV.getPointer(CGF))) - if (GEP->use_empty()) - GEP->eraseFromParent(); - } - - // Deactivate all the partial cleanups in reverse order, which - // generally means popping them. - assert((cleanupDominator || cleanups.empty()) && - "Missing cleanupDominator before deactivating cleanup blocks"); - for (unsigned i = cleanups.size(); i != 0; --i) - CGF.DeactivateCleanupBlock(cleanups[i-1], cleanupDominator); - - // Destroy the placeholder if we made one. - if (cleanupDominator) - cleanupDominator->eraseFromParent(); + } } void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, @@ -1833,9 +1809,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, // destPtr is an array*. Construct an elementType* by drilling down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = {zero, zero}; - llvm::Value *begin = Builder.CreateInBoundsGEP( - destPtr.getElementType(), destPtr.getPointer(), indices, - "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(), + destPtr.emitRawPointer(CGF), + indices, "arrayinit.begin"); // Prepare to special-case multidimensional array initialization: we avoid // emitting multiple destructor loops in that case. @@ -1887,7 +1863,7 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, if (InnerLoop) { // If the subexpression is an ArrayInitLoopExpr, share its cleanup. auto elementSlot = AggValueSlot::forLValue( - elementLV, CGF, AggValueSlot::IsDestructed, + elementLV, AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap); AggExprEmitter(CGF, elementSlot, false) @@ -1965,7 +1941,7 @@ static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) { if (Field->getType()->isIncompleteArrayType() || ILEElement == ILE->getNumInits()) break; - if (Field->isUnnamedBitfield()) + if (Field->isUnnamedBitField()) continue; const Expr *E = ILE->getInit(ILEElement++); @@ -2052,18 +2028,29 @@ LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) { assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!"); Address Temp = CreateMemTemp(E->getType()); LValue LV = MakeAddrLValue(Temp, E->getType()); - EmitAggExpr(E, AggValueSlot::forLValue( - LV, *this, AggValueSlot::IsNotDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap)); + EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, + AggValueSlot::DoesNotOverlap)); return LV; } +void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, + const LValue &Src, + ExprValueKind SrcKind) { + return AggExprEmitter(*this, Dest, Dest.isIgnored()) + .EmitFinalDestCopy(Type, Src, SrcKind); +} + AggValueSlot::Overlap_t CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) { if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType()) return AggValueSlot::DoesNotOverlap; + // Empty fields can overlap earlier fields. + if (FD->getType()->getAsCXXRecordDecl()->isEmpty()) + return AggValueSlot::MayOverlap; + // If the field lies entirely within the enclosing class's nvsize, its tail // padding cannot overlap any already-initialized object. (The only subobjects // with greater addresses that might already be initialized are vbases.) @@ -2086,6 +2073,10 @@ AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit( if (IsVirtual) return AggValueSlot::MayOverlap; + // Empty bases can overlap earlier bases. + if (BaseRD->isEmpty()) + return AggValueSlot::MayOverlap; + // If the base class is laid out entirely within the nvsize of the derived // class, its tail padding cannot yet be initialized, so we can issue // stores at the full width of the base class. @@ -2104,8 +2095,8 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty, bool isVolatile) { assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex"); - Address DestPtr = Dest.getAddress(*this); - Address SrcPtr = Src.getAddress(*this); + Address DestPtr = Dest.getAddress(); + Address SrcPtr = Src.getAddress(); if (getLangOpts().CPlusPlus) { if (const RecordType *RT = Ty->getAs<RecordType>()) { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp index d136bfc37278..8eb6ab7381ac 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp @@ -142,7 +142,7 @@ RValue CodeGenFunction::EmitCXXPseudoDestructorExpr( BaseQuals = PTy->getPointeeType().getQualifiers(); } else { LValue BaseLV = EmitLValue(BaseExpr); - BaseValue = BaseLV.getAddress(*this); + BaseValue = BaseLV.getAddress(); QualType BaseTy = BaseExpr->getType(); BaseQuals = BaseTy.getQualifiers(); } @@ -280,7 +280,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); - This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo); + This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(), + BaseInfo, TBAAInfo); } else { This = EmitLValue(Base); } @@ -297,7 +298,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( /*ImplicitParamTy=*/QualType(), CE, Args, nullptr); EmitCXXConstructorCall(Ctor, Ctor_Complete, /*ForVirtualBase=*/false, - /*Delegating=*/false, This.getAddress(*this), Args, + /*Delegating=*/false, This.getAddress(), Args, AggValueSlot::DoesNotOverlap, CE->getExprLoc(), /*NewPointerIsChecked=*/false); return RValue::get(nullptr); @@ -353,10 +354,12 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(IOA)) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, - This.getPointer(*this), - C.getRecordType(CalleeDecl->getParent()), - /*Alignment=*/CharUnits::Zero(), SkippedChecks); + + if (sanitizePerformTypeCheck()) + EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, + This.emitRawPointer(*this), + C.getRecordType(CalleeDecl->getParent()), + /*Alignment=*/CharUnits::Zero(), SkippedChecks); // C++ [class.virtual]p12: // Explicit qualification with the scope operator (5.1) suppresses the @@ -372,7 +375,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( assert(ReturnValue.isNull() && "Destructor shouldn't have return value"); if (UseVirtualCall) { CGM.getCXXABI().EmitVirtualDestructorCall(*this, Dtor, Dtor_Complete, - This.getAddress(*this), + This.getAddress(), cast<CXXMemberCallExpr>(CE)); } else { GlobalDecl GD(Dtor, Dtor_Complete); @@ -400,14 +403,14 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( CGCallee Callee; if (UseVirtualCall) { - Callee = CGCallee::forVirtual(CE, MD, This.getAddress(*this), Ty); + Callee = CGCallee::forVirtual(CE, MD, This.getAddress(), Ty); } else { if (SanOpts.has(SanitizerKind::CFINVCall) && MD->getParent()->isDynamicClass()) { llvm::Value *VTable; const CXXRecordDecl *RD; std::tie(VTable, RD) = CGM.getCXXABI().LoadVTablePtr( - *this, This.getAddress(*this), CalleeDecl->getParent()); + *this, This.getAddress(), CalleeDecl->getParent()); EmitVTablePtrCheckForCall(RD, VTable, CFITCK_NVCall, CE->getBeginLoc()); } @@ -426,7 +429,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (MD->isVirtual()) { Address NewThisAddr = CGM.getCXXABI().adjustThisArgumentForVirtualFunctionCall( - *this, CalleeDecl, This.getAddress(*this), UseVirtualCall); + *this, CalleeDecl, This.getAddress(), UseVirtualCall); This.setAddress(NewThisAddr); } @@ -453,9 +456,9 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, if (BO->getOpcode() == BO_PtrMemI) This = EmitPointerWithAlignment(BaseExpr, nullptr, nullptr, KnownNonNull); else - This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this); + This = EmitLValue(BaseExpr, KnownNonNull).getAddress(); - EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), + EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this), QualType(MPT->getClass(), 0)); // Get the member function pointer. @@ -1005,8 +1008,8 @@ void CodeGenFunction::EmitNewArrayInitializer( const Expr *Init = E->getInitializer(); Address EndOfInit = Address::invalid(); QualType::DestructionKind DtorKind = ElementType.isDestructedType(); - EHScopeStack::stable_iterator Cleanup; - llvm::Instruction *CleanupDominator = nullptr; + CleanupDeactivationScope deactivation(*this); + bool pushedCleanup = false; CharUnits ElementSize = getContext().getTypeSizeInChars(ElementType); CharUnits ElementAlign = @@ -1073,8 +1076,7 @@ void CodeGenFunction::EmitNewArrayInitializer( // Move past these elements. InitListElements = cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe()) - ->getSize() - .getZExtValue(); + ->getZExtSize(); CurPtr = Builder.CreateConstInBoundsGEP( CurPtr, InitListElements, "string.init.end"); @@ -1103,18 +1105,24 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Enter a partial-destruction Cleanup if necessary. - if (needsEHCleanup(DtorKind)) { + if (DtorKind) { + AllocaTrackerRAII AllocaTracker(*this); // In principle we could tell the Cleanup where we are more // directly, but the control flow can get so varied here that it // would actually be quite complex. Therefore we go through an // alloca. + llvm::Instruction *DominatingIP = + Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(Int8PtrTy)); EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(), "array.init.end"); - CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit); - pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit, - ElementType, ElementAlign, + pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this), + EndOfInit, ElementType, ElementAlign, getDestroyer(DtorKind)); - Cleanup = EHStack.stable_begin(); + cast<EHCleanupScope>(*EHStack.find(EHStack.stable_begin())) + .AddAuxAllocas(AllocaTracker.Take()); + DeferredDeactivationCleanupStack.push_back( + {EHStack.stable_begin(), DominatingIP}); + pushedCleanup = true; } CharUnits StartAlign = CurPtr.getAlignment(); @@ -1124,16 +1132,17 @@ void CodeGenFunction::EmitNewArrayInitializer( // element. TODO: some of these stores can be trivially // observed to be unnecessary. if (EndOfInit.isValid()) { - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); } // FIXME: If the last initializer is an incomplete initializer list for // an array, and we have an array filler, we can fold together the two // initialization loops. StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr, AggValueSlot::DoesNotOverlap); - CurPtr = Address(Builder.CreateInBoundsGEP( - CurPtr.getElementType(), CurPtr.getPointer(), - Builder.getSize(1), "array.exp.next"), + CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(), + CurPtr.emitRawPointer(*this), + Builder.getSize(1), + "array.exp.next"), CurPtr.getElementType(), StartAlign.alignmentAtOffset((++i) * ElementSize)); } @@ -1160,9 +1169,6 @@ void CodeGenFunction::EmitNewArrayInitializer( // initialization. llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements); if (ConstNum && ConstNum->getZExtValue() <= InitListElements) { - // If there was a Cleanup, deactivate it. - if (CleanupDominator) - DeactivateCleanupBlock(Cleanup, CleanupDominator); return; } @@ -1187,7 +1193,7 @@ void CodeGenFunction::EmitNewArrayInitializer( // FIXME: Share this cleanup with the constructor call emission rather than // having it create a cleanup of its own. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); // Emit a constructor call loop to initialize the remaining elements. if (InitListElements) @@ -1231,7 +1237,7 @@ void CodeGenFunction::EmitNewArrayInitializer( if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RType->getDecl())) NumElements = CXXRD->getNumBases(); for (auto *Field : RType->getDecl()->fields()) - if (!Field->isUnnamedBitfield()) + if (!Field->isUnnamedBitField()) ++NumElements; // FIXME: Recurse into nested InitListExprs. if (ILE->getNumInits() == NumElements) @@ -1250,15 +1256,15 @@ void CodeGenFunction::EmitNewArrayInitializer( llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end"); // Find the end of the array, hoisted out of the loop. - llvm::Value *EndPtr = - Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(), - NumElements, "array.end"); + llvm::Value *EndPtr = Builder.CreateInBoundsGEP( + BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements, + "array.end"); // If the number of elements isn't constant, we have to now check if there is // anything left to initialize. if (!ConstNum) { - llvm::Value *IsEmpty = - Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty"); + llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this), + EndPtr, "array.isempty"); Builder.CreateCondBr(IsEmpty, ContBB, LoopBB); } @@ -1268,21 +1274,23 @@ void CodeGenFunction::EmitNewArrayInitializer( // Set up the current-element phi. llvm::PHINode *CurPtrPhi = Builder.CreatePHI(CurPtr.getType(), 2, "array.cur"); - CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB); + CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB); CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign); // Store the new Cleanup position for irregular Cleanups. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); // Enter a partial-destruction Cleanup if necessary. - if (!CleanupDominator && needsEHCleanup(DtorKind)) { - pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(), - ElementType, ElementAlign, - getDestroyer(DtorKind)); - Cleanup = EHStack.stable_begin(); - CleanupDominator = Builder.CreateUnreachable(); + if (!pushedCleanup && needsEHCleanup(DtorKind)) { + llvm::Instruction *DominatingIP = + Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(Int8PtrTy)); + pushRegularPartialArrayCleanup(BeginPtr.emitRawPointer(*this), + CurPtr.emitRawPointer(*this), ElementType, + ElementAlign, getDestroyer(DtorKind)); + DeferredDeactivationCleanupStack.push_back( + {EHStack.stable_begin(), DominatingIP}); } // Emit the initializer into this element. @@ -1290,15 +1298,11 @@ void CodeGenFunction::EmitNewArrayInitializer( AggValueSlot::DoesNotOverlap); // Leave the Cleanup if we entered one. - if (CleanupDominator) { - DeactivateCleanupBlock(Cleanup, CleanupDominator); - CleanupDominator->eraseFromParent(); - } + deactivation.ForceDeactivate(); // Advance to the next element by adjusting the pointer type as necessary. - llvm::Value *NextPtr = - Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1, - "array.next"); + llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32( + ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1423,6 +1427,7 @@ namespace { }; unsigned NumPlacementArgs : 31; + LLVM_PREFERRED_TYPE(bool) unsigned PassAlignmentToPlacementDelete : 1; const FunctionDecl *OperatorDelete; ValueTy Ptr; @@ -1523,14 +1528,9 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup; - DirectCleanup *Cleanup = CGF.EHStack - .pushCleanupWithExtra<DirectCleanup>(EHCleanup, - E->getNumPlacementArgs(), - E->getOperatorDelete(), - NewPtr.getPointer(), - AllocSize, - E->passAlignment(), - AllocAlign); + DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra<DirectCleanup>( + EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(), + NewPtr.emitRawPointer(CGF), AllocSize, E->passAlignment(), AllocAlign); for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) { auto &Arg = NewArgs[I + NumNonPlacementArgs]; Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty); @@ -1541,7 +1541,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, // Otherwise, we need to save all this stuff. DominatingValue<RValue>::saved_type SavedNewPtr = - DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer())); + DominatingValue<RValue>::save(CGF, RValue::get(NewPtr, CGF)); DominatingValue<RValue>::saved_type SavedAllocSize = DominatingValue<RValue>::save(CGF, RValue::get(AllocSize)); @@ -1590,8 +1590,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { isa<StringLiteral>(IgnoreParen) || isa<ObjCEncodeExpr>(IgnoreParen)) { minElements = cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe()) - ->getSize() - .getZExtValue(); + ->getZExtSize(); } else if (ILE || CPLIE) { minElements = ILE ? ILE->getNumInits() : CPLIE->getInitExprs().size(); } @@ -1619,14 +1618,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // In these cases, discard the computed alignment and use the // formal alignment of the allocated type. if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl) - allocation = allocation.withAlignment(allocAlign); + allocation.setAlignment(allocAlign); // Set up allocatorArgs for the call to operator delete if it's not // the reserved global operator. if (E->getOperatorDelete() && !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) { allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType()); - allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType()); + allocatorArgs.add(RValue::get(allocation, *this), arg->getType()); } } else { @@ -1714,8 +1713,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull"); contBB = createBasicBlock("new.cont"); - llvm::Value *isNull = - Builder.CreateIsNull(allocation.getPointer(), "new.isnull"); + llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull"); Builder.CreateCondBr(isNull, contBB, notNullBB); EmitBlock(notNullBB); } @@ -1761,12 +1759,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { SkippedChecks.set(SanitizerKind::Null, nullCheck); EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(), - result.getPointer(), allocType, result.getAlignment(), - SkippedChecks, numElements); + result, allocType, result.getAlignment(), SkippedChecks, + numElements); EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); - llvm::Value *resultPtr = result.getPointer(); + llvm::Value *resultPtr = result.emitRawPointer(*this); if (E->isArray()) { // NewPtr is a pointer to the base element type. If we're // allocating an array of arrays, we'll need to cast back to the @@ -1910,7 +1908,8 @@ static void EmitDestroyingObjectDelete(CodeGenFunction &CGF, CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType, Dtor); else - CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType); + CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF), + ElementType); } /// Emit the code for deleting a single object. @@ -1926,8 +1925,7 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // dynamic type, the static type shall be a base class of the dynamic type // of the object to be deleted and the static type shall have a virtual // destructor or the behavior is undefined. - CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, - DE->getExprLoc(), Ptr.getPointer(), + CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr, ElementType); const FunctionDecl *OperatorDelete = DE->getOperatorDelete(); @@ -1976,9 +1974,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // Make sure that we call delete even if the dtor throws. // This doesn't have to a conditional cleanup because we're going // to pop it off in a second. - CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup, - Ptr.getPointer(), - OperatorDelete, ElementType); + CGF.EHStack.pushCleanup<CallObjectDelete>( + NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType); if (Dtor) CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, @@ -2065,7 +2062,7 @@ static void EmitArrayDelete(CodeGenFunction &CGF, CharUnits elementAlign = deletedPtr.getAlignment().alignmentOfArrayElement(elementSize); - llvm::Value *arrayBegin = deletedPtr.getPointer(); + llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF); llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP( deletedPtr.getElementType(), arrayBegin, numElements, "delete.end"); @@ -2096,7 +2093,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull"); llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end"); - llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull"); + llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull"); Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull); EmitBlock(DeleteNotNull); @@ -2131,10 +2128,8 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { GEP.push_back(Zero); } - Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(), - Ptr.getPointer(), GEP, "del.first"), - ConvertTypeForMem(DeleteTy), Ptr.getAlignment(), - Ptr.isKnownNonNull()); + Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, ConvertTypeForMem(DeleteTy), + Ptr.getAlignment(), "del.first"); } assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType()); @@ -2148,42 +2143,11 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { } } -static bool isGLValueFromPointerDeref(const Expr *E) { - E = E->IgnoreParens(); - - if (const auto *CE = dyn_cast<CastExpr>(E)) { - if (!CE->getSubExpr()->isGLValue()) - return false; - return isGLValueFromPointerDeref(CE->getSubExpr()); - } - - if (const auto *OVE = dyn_cast<OpaqueValueExpr>(E)) - return isGLValueFromPointerDeref(OVE->getSourceExpr()); - - if (const auto *BO = dyn_cast<BinaryOperator>(E)) - if (BO->getOpcode() == BO_Comma) - return isGLValueFromPointerDeref(BO->getRHS()); - - if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(E)) - return isGLValueFromPointerDeref(ACO->getTrueExpr()) || - isGLValueFromPointerDeref(ACO->getFalseExpr()); - - // C++11 [expr.sub]p1: - // The expression E1[E2] is identical (by definition) to *((E1)+(E2)) - if (isa<ArraySubscriptExpr>(E)) - return true; - - if (const auto *UO = dyn_cast<UnaryOperator>(E)) - if (UO->getOpcode() == UO_Deref) - return true; - - return false; -} - static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, - llvm::Type *StdTypeInfoPtrTy) { + llvm::Type *StdTypeInfoPtrTy, + bool HasNullCheck) { // Get the vtable pointer. - Address ThisPtr = CGF.EmitLValue(E).getAddress(CGF); + Address ThisPtr = CGF.EmitLValue(E).getAddress(); QualType SrcRecordTy = E->getType(); @@ -2192,23 +2156,18 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, // destruction and the static type of the operand is neither the constructor // or destructor’s class nor one of its bases, the behavior is undefined. CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(), - ThisPtr.getPointer(), SrcRecordTy); + ThisPtr, SrcRecordTy); - // C++ [expr.typeid]p2: - // If the glvalue expression is obtained by applying the unary * operator to - // a pointer and the pointer is a null pointer value, the typeid expression - // throws the std::bad_typeid exception. - // - // However, this paragraph's intent is not clear. We choose a very generous - // interpretation which implores us to consider comma operators, conditional - // operators, parentheses and other such constructs. - if (CGF.CGM.getCXXABI().shouldTypeidBeNullChecked( - isGLValueFromPointerDeref(E), SrcRecordTy)) { + // Whether we need an explicit null pointer check. For example, with the + // Microsoft ABI, if this is a call to __RTtypeid, the null pointer check and + // exception throw is inside the __RTtypeid(nullptr) call + if (HasNullCheck && + CGF.CGM.getCXXABI().shouldTypeidBeNullChecked(SrcRecordTy)) { llvm::BasicBlock *BadTypeidBlock = CGF.createBasicBlock("typeid.bad_typeid"); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end"); - llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer()); + llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr); CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock); CGF.EmitBlock(BadTypeidBlock); @@ -2221,7 +2180,12 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, } llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) { - llvm::Type *PtrTy = llvm::PointerType::getUnqual(getLLVMContext()); + // Ideally, we would like to use GlobalsInt8PtrTy here, however, we cannot, + // primarily because the result of applying typeid is a value of type + // type_info, which is declared & defined by the standard library + // implementation and expects to operate on the generic (default) AS. + // https://reviews.llvm.org/D157452 has more context, and a possible solution. + llvm::Type *PtrTy = Int8PtrTy; LangAS GlobAS = CGM.GetGlobalVarAddressSpace(nullptr); auto MaybeASCast = [=](auto &&TypeInfo) { @@ -2244,7 +2208,8 @@ llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) { // type) to which the glvalue refers. // If the operand is already most derived object, no need to look up vtable. if (E->isPotentiallyEvaluated() && !E->isMostDerived(getContext())) - return EmitTypeidFromVTable(*this, E->getExprOperand(), PtrTy); + return EmitTypeidFromVTable(*this, E->getExprOperand(), PtrTy, + E->hasNullCheck()); QualType OperandTy = E->getExprOperand()->getType(); return MaybeASCast(CGM.GetAddrOfRTTIDescriptor(OperandTy)); @@ -2294,8 +2259,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, // construction or destruction and the static type of the operand is not a // pointer to or object of the constructor or destructor’s own class or one // of its bases, the dynamic_cast results in undefined behavior. - EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(), - SrcRecordTy); + EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy); if (DCE->isAlwaysNull()) { if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) { @@ -2330,7 +2294,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, CastNull = createBasicBlock("dynamic_cast.null"); CastNotNull = createBasicBlock("dynamic_cast.notnull"); - llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer()); + llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprComplex.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprComplex.cpp index 839fe16cd772..4d45f6d64c1c 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprComplex.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprComplex.cpp @@ -28,6 +28,10 @@ using namespace CodeGen; // Complex Expression Emitter //===----------------------------------------------------------------------===// +namespace llvm { +extern cl::opt<bool> EnableSingleByteCoverage; +} // namespace llvm + typedef CodeGenFunction::ComplexPairTy ComplexPairTy; /// Return the complex type that we are meant to emit. @@ -47,11 +51,12 @@ class ComplexExprEmitter CGBuilderTy &Builder; bool IgnoreReal; bool IgnoreImag; -public: - ComplexExprEmitter(CodeGenFunction &cgf, bool ir=false, bool ii=false) - : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii) { - } + bool FPHasBeenPromoted; +public: + ComplexExprEmitter(CodeGenFunction &cgf, bool ir = false, bool ii = false) + : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii), + FPHasBeenPromoted(false) {} //===--------------------------------------------------------------------===// // Utilities @@ -283,9 +288,62 @@ public: ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName, const BinOpInfo &Op); - QualType getPromotionType(QualType Ty) { + QualType GetHigherPrecisionFPType(QualType ElementType) { + const auto *CurrentBT = cast<BuiltinType>(ElementType); + switch (CurrentBT->getKind()) { + case BuiltinType::Kind::Float16: + return CGF.getContext().FloatTy; + case BuiltinType::Kind::Float: + case BuiltinType::Kind::BFloat16: + return CGF.getContext().DoubleTy; + case BuiltinType::Kind::Double: + return CGF.getContext().LongDoubleTy; + default: + return ElementType; + } + } + + QualType HigherPrecisionTypeForComplexArithmetic(QualType ElementType, + bool IsDivOpCode) { + QualType HigherElementType = GetHigherPrecisionFPType(ElementType); + const llvm::fltSemantics &ElementTypeSemantics = + CGF.getContext().getFloatTypeSemantics(ElementType); + const llvm::fltSemantics &HigherElementTypeSemantics = + CGF.getContext().getFloatTypeSemantics(HigherElementType); + // Check that the promoted type can handle the intermediate values without + // overflowing. This can be interpreted as: + // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <= + // LargerType.LargestFiniteVal. + // In terms of exponent it gives this formula: + // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal + // doubles the exponent of SmallerType.LargestFiniteVal) + if (llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 <= + llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) { + FPHasBeenPromoted = true; + return CGF.getContext().getComplexType(HigherElementType); + } else { + DiagnosticsEngine &Diags = CGF.CGM.getDiags(); + Diags.Report(diag::warn_next_larger_fp_type_same_size_than_fp); + return QualType(); + } + } + + QualType getPromotionType(FPOptionsOverride Features, QualType Ty, + bool IsDivOpCode = false) { if (auto *CT = Ty->getAs<ComplexType>()) { QualType ElementType = CT->getElementType(); + bool IsFloatingType = ElementType->isFloatingType(); + bool IsComplexRangePromoted = CGF.getLangOpts().getComplexRange() == + LangOptions::ComplexRangeKind::CX_Promoted; + bool HasNoComplexRangeOverride = !Features.hasComplexRangeOverride(); + bool HasMatchingComplexRange = Features.hasComplexRangeOverride() && + Features.getComplexRangeOverride() == + CGF.getLangOpts().getComplexRange(); + + if (IsDivOpCode && IsFloatingType && IsComplexRangePromoted && + (HasNoComplexRangeOverride || HasMatchingComplexRange)) + return HigherPrecisionTypeForComplexArithmetic(ElementType, + IsDivOpCode); if (ElementType.UseExcessPrecision(CGF.getContext())) return CGF.getContext().getComplexType(CGF.getContext().FloatTy); } @@ -296,11 +354,12 @@ public: #define HANDLEBINOP(OP) \ ComplexPairTy VisitBin##OP(const BinaryOperator *E) { \ - QualType promotionTy = getPromotionType(E->getType()); \ + QualType promotionTy = getPromotionType( \ + E->getStoredFPFeaturesOrDefault(), E->getType(), \ + (E->getOpcode() == BinaryOperatorKind::BO_Div) ? true : false); \ ComplexPairTy result = EmitBin##OP(EmitBinOps(E, promotionTy)); \ if (!promotionTy.isNull()) \ - result = \ - CGF.EmitUnPromotedValue(result, E->getType()); \ + result = CGF.EmitUnPromotedValue(result, E->getType()); \ return result; \ } @@ -354,6 +413,10 @@ public: ComplexPairTy VisitAtomicExpr(AtomicExpr *E) { return CGF.EmitAtomicExpr(E).getComplexVal(); } + + ComplexPairTy VisitPackIndexingExpr(PackIndexingExpr *E) { + return Visit(E->getSelectedExpr()); + } }; } // end anonymous namespace. @@ -379,7 +442,7 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, if (lvalue.getType()->isAtomicType()) return CGF.EmitAtomicLoad(lvalue, loc).getComplexVal(); - Address SrcPtr = lvalue.getAddress(CGF); + Address SrcPtr = lvalue.getAddress(); bool isVolatile = lvalue.isVolatileQualified(); llvm::Value *Real = nullptr, *Imag = nullptr; @@ -405,7 +468,7 @@ void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue, (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue))) return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit); - Address Ptr = lvalue.getAddress(CGF); + Address Ptr = lvalue.getAddress(); Address RealPtr = CGF.emitAddrOfRealComponent(Ptr, lvalue.getType()); Address ImagPtr = CGF.emitAddrOfImagComponent(Ptr, lvalue.getType()); @@ -496,14 +559,14 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, case CK_LValueBitCast: { LValue origLV = CGF.EmitLValue(Op); - Address V = origLV.getAddress(CGF).withElementType(CGF.ConvertType(DestTy)); + Address V = origLV.getAddress().withElementType(CGF.ConvertType(DestTy)); return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy), Op->getExprLoc()); } case CK_LValueToRValueBitCast: { LValue SourceLVal = CGF.EmitLValue(Op); - Address Addr = SourceLVal.getAddress(CGF).withElementType( - CGF.ConvertTypeForMem(DestTy)); + Address Addr = + SourceLVal.getAddress().withElementType(CGF.ConvertTypeForMem(DestTy)); LValue DestLV = CGF.MakeAddrLValue(Addr, DestTy); DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo()); return EmitLoadOfLValue(DestLV, Op->getExprLoc()); @@ -560,6 +623,8 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, case CK_FixedPointToIntegral: case CK_IntegralToFixedPoint: case CK_MatrixCast: + case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: llvm_unreachable("invalid cast kind for complex value"); case CK_FloatingRealToComplex: @@ -584,9 +649,12 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, ComplexPairTy ComplexExprEmitter::VisitUnaryPlus(const UnaryOperator *E, QualType PromotionType) { - QualType promotionTy = PromotionType.isNull() - ? getPromotionType(E->getSubExpr()->getType()) - : PromotionType; + E->hasStoredFPFeatures(); + QualType promotionTy = + PromotionType.isNull() + ? getPromotionType(E->getStoredFPFeaturesOrDefault(), + E->getSubExpr()->getType()) + : PromotionType; ComplexPairTy result = VisitPlus(E, promotionTy); if (!promotionTy.isNull()) return CGF.EmitUnPromotedValue(result, E->getSubExpr()->getType()); @@ -604,9 +672,11 @@ ComplexPairTy ComplexExprEmitter::VisitPlus(const UnaryOperator *E, ComplexPairTy ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *E, QualType PromotionType) { - QualType promotionTy = PromotionType.isNull() - ? getPromotionType(E->getSubExpr()->getType()) - : PromotionType; + QualType promotionTy = + PromotionType.isNull() + ? getPromotionType(E->getStoredFPFeaturesOrDefault(), + E->getSubExpr()->getType()) + : PromotionType; ComplexPairTy result = VisitMinus(E, promotionTy); if (!promotionTy.isNull()) return CGF.EmitUnPromotedValue(result, E->getSubExpr()->getType()); @@ -760,8 +830,6 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { // // But we can fold away components which would be zero due to a real // operand according to C11 Annex G.5.1p2. - // FIXME: C11 also provides for imaginary types which would allow folding - // still more of this within the type system. CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures); if (Op.LHS.second && Op.RHS.second) { @@ -785,8 +853,9 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { ResR = Builder.CreateFSub(AC, BD, "mul_r"); ResI = Builder.CreateFAdd(AD, BC, "mul_i"); - if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited || - Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Basic || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted) return ComplexPairTy(ResR, ResI); // Emit the test for the real part becoming NaN and create a branch to @@ -798,8 +867,7 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { llvm::BasicBlock *OrigBB = Branch->getParent(); // Give hint that we very much don't expect to see NaNs. - // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp - llvm::MDNode *BrWeight = MDHelper.createBranchWeights(1, (1U << 20) - 1); + llvm::MDNode *BrWeight = MDHelper.createUnlikelyBranchWeights(); Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight); // Now test the imaginary part and create its branch. @@ -977,22 +1045,21 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { llvm::Value *OrigLHSi = LHSi; if (!LHSi) LHSi = llvm::Constant::getNullValue(RHSi->getType()); - if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved || + (Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted && + !FPHasBeenPromoted)) return EmitRangeReductionDiv(LHSr, LHSi, RHSr, RHSi); - else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited) + else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Basic || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted) return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi); - else if (!CGF.getLangOpts().FastMath || - // '-ffast-math' is used in the command line but followed by an - // '-fno-cx-limited-range'. - Op.FPFeatures.getComplexRange() == LangOptions::CX_Full) { + // '-ffast-math' is used in the command line but followed by an + // '-fno-cx-limited-range' or '-fcomplex-arithmetic=full'. + else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Full) { LHSi = OrigLHSi; // If we have a complex operand on the RHS and FastMath is not allowed, we // delegate to a libcall to handle all of the complexities and minimize // underflow/overflow cases. When FastMath is allowed we construct the // divide inline using the same algorithm as for integer operands. - // - // FIXME: We would be able to avoid the libcall in many places if we - // supported imaginary types in addition to complex types. BinOpInfo LibCallOp = Op; // If LHS was a real, supply a null imaginary part. if (!LHSi) @@ -1164,13 +1231,15 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E, // __block variables need to have the rhs evaluated first, plus this should // improve codegen a little. QualType PromotionTypeCR; - PromotionTypeCR = getPromotionType(E->getComputationResultType()); + PromotionTypeCR = getPromotionType(E->getStoredFPFeaturesOrDefault(), + E->getComputationResultType()); if (PromotionTypeCR.isNull()) PromotionTypeCR = E->getComputationResultType(); OpInfo.Ty = PromotionTypeCR; QualType ComplexElementTy = OpInfo.Ty->castAs<ComplexType>()->getElementType(); - QualType PromotionTypeRHS = getPromotionType(E->getRHS()->getType()); + QualType PromotionTypeRHS = getPromotionType( + E->getStoredFPFeaturesOrDefault(), E->getRHS()->getType()); // The RHS should have been converted to the computation type. if (E->getRHS()->getType()->isRealFloatingType()) { @@ -1198,7 +1267,8 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E, // Load from the l-value and convert it. SourceLocation Loc = E->getExprLoc(); - QualType PromotionTypeLHS = getPromotionType(E->getComputationLHSType()); + QualType PromotionTypeLHS = getPromotionType( + E->getStoredFPFeaturesOrDefault(), E->getComputationLHSType()); if (LHSTy->isAnyComplexType()) { ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, Loc); if (!PromotionTypeLHS.isNull()) @@ -1325,7 +1395,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { eval.begin(CGF); CGF.EmitBlock(LHSBlock); - CGF.incrementProfileCounter(E); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E->getTrueExpr()); + else + CGF.incrementProfileCounter(E); + ComplexPairTy LHS = Visit(E->getTrueExpr()); LHSBlock = Builder.GetInsertBlock(); CGF.EmitBranch(ContBlock); @@ -1333,9 +1407,13 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { eval.begin(CGF); CGF.EmitBlock(RHSBlock); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E->getFalseExpr()); ComplexPairTy RHS = Visit(E->getFalseExpr()); RHSBlock = Builder.GetInsertBlock(); CGF.EmitBlock(ContBlock); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E); eval.end(CGF); // Create a PHI node for the real part. @@ -1381,9 +1459,9 @@ ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) { ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) { Address ArgValue = Address::invalid(); - Address ArgPtr = CGF.EmitVAArg(E, ArgValue); + RValue RV = CGF.EmitVAArg(E, ArgValue); - if (!ArgPtr.isValid()) { + if (!ArgValue.isValid()) { CGF.ErrorUnsupported(E, "complex va_arg expression"); llvm::Type *EltTy = CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType()); @@ -1391,8 +1469,7 @@ ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) { return ComplexPairTy(U, U); } - return EmitLoadOfLValue(CGF.MakeAddrLValue(ArgPtr, E->getType()), - E->getExprLoc()); + return RV.getComplexVal(); } //===----------------------------------------------------------------------===// diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp index 604e3958161d..f22321f0e738 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "ABIInfoImpl.h" #include "CGCXXABI.h" #include "CGObjCRuntime.h" #include "CGRecordLayout.h" @@ -393,7 +394,7 @@ bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) { static llvm::Constant * EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, - llvm::Type *CommonElementType, unsigned ArrayBound, + llvm::Type *CommonElementType, uint64_t ArrayBound, SmallVectorImpl<llvm::Constant *> &Elements, llvm::Constant *Filler); @@ -564,12 +565,13 @@ class ConstStructBuilder { public: static llvm::Constant *BuildStruct(ConstantEmitter &Emitter, - InitListExpr *ILE, QualType StructTy); + const InitListExpr *ILE, + QualType StructTy); static llvm::Constant *BuildStruct(ConstantEmitter &Emitter, const APValue &Value, QualType ValTy); static bool UpdateStruct(ConstantEmitter &Emitter, ConstantAggregateBuilder &Const, CharUnits Offset, - InitListExpr *Updater); + const InitListExpr *Updater); private: ConstStructBuilder(ConstantEmitter &Emitter, @@ -584,9 +586,9 @@ private: bool AllowOverwrite = false); bool AppendBitField(const FieldDecl *Field, uint64_t FieldOffset, - llvm::ConstantInt *InitExpr, bool AllowOverwrite = false); + llvm::Constant *InitExpr, bool AllowOverwrite = false); - bool Build(InitListExpr *ILE, bool AllowOverwrite); + bool Build(const InitListExpr *ILE, bool AllowOverwrite); bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase, const CXXRecordDecl *VTableClass, CharUnits BaseOffset); llvm::Constant *Finalize(QualType Ty); @@ -608,9 +610,25 @@ bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars, return Builder.add(InitCst, StartOffset + FieldOffsetInChars, AllowOverwrite); } -bool ConstStructBuilder::AppendBitField( - const FieldDecl *Field, uint64_t FieldOffset, llvm::ConstantInt *CI, - bool AllowOverwrite) { +bool ConstStructBuilder::AppendBitField(const FieldDecl *Field, + uint64_t FieldOffset, llvm::Constant *C, + bool AllowOverwrite) { + + llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(C); + if (!CI) { + // Constants for long _BitInt types are sometimes split into individual + // bytes. Try to fold these back into an integer constant. If that doesn't + // work out, then we are trying to initialize a bitfield with a non-trivial + // constant, this must require run-time code. + llvm::Type *LoadType = + CGM.getTypes().convertTypeForLoadStore(Field->getType(), C->getType()); + llvm::Constant *FoldedConstant = llvm::ConstantFoldLoadFromConst( + C, LoadType, llvm::APInt::getZero(32), CGM.getDataLayout()); + CI = dyn_cast_if_present<llvm::ConstantInt>(FoldedConstant); + if (!CI) + return false; + } + const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(Field->getParent()); const CGBitFieldInfo &Info = RL.getBitFieldInfo(Field); @@ -635,7 +653,7 @@ bool ConstStructBuilder::AppendBitField( static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, ConstantAggregateBuilder &Const, CharUnits Offset, QualType Type, - InitListExpr *Updater) { + const InitListExpr *Updater) { if (Type->isRecordType()) return ConstStructBuilder::UpdateStruct(Emitter, Const, Offset, Updater); @@ -647,7 +665,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, llvm::Type *ElemTy = Emitter.CGM.getTypes().ConvertTypeForMem(ElemType); llvm::Constant *FillC = nullptr; - if (Expr *Filler = Updater->getArrayFiller()) { + if (const Expr *Filler = Updater->getArrayFiller()) { if (!isa<NoInitExpr>(Filler)) { FillC = Emitter.tryEmitAbstractForMemory(Filler, ElemType); if (!FillC) @@ -656,9 +674,9 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, } unsigned NumElementsToUpdate = - FillC ? CAT->getSize().getZExtValue() : Updater->getNumInits(); + FillC ? CAT->getZExtSize() : Updater->getNumInits(); for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) { - Expr *Init = nullptr; + const Expr *Init = nullptr; if (I < Updater->getNumInits()) Init = Updater->getInit(I); @@ -667,7 +685,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, return false; } else if (!Init || isa<NoInitExpr>(Init)) { continue; - } else if (InitListExpr *ChildILE = dyn_cast<InitListExpr>(Init)) { + } else if (const auto *ChildILE = dyn_cast<InitListExpr>(Init)) { if (!EmitDesignatedInitUpdater(Emitter, Const, Offset, ElemType, ChildILE)) return false; @@ -683,7 +701,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, return true; } -bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) { +bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) { RecordDecl *RD = ILE->getType()->castAs<RecordType>()->getDecl(); const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD); @@ -706,20 +724,20 @@ bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) { continue; // Don't emit anonymous bitfields. - if (Field->isUnnamedBitfield()) + if (Field->isUnnamedBitField()) continue; // Get the initializer. A struct can include fields without initializers, // we just use explicit null values for them. - Expr *Init = nullptr; + const Expr *Init = nullptr; if (ElementNo < ILE->getNumInits()) Init = ILE->getInit(ElementNo++); - if (Init && isa<NoInitExpr>(Init)) + if (isa_and_nonnull<NoInitExpr>(Init)) continue; // Zero-sized fields are not emitted, but their initializers may still // prevent emission of this struct as a constant. - if (Field->isZeroSize(CGM.getContext())) { + if (isEmptyFieldForLayout(CGM.getContext(), Field)) { if (Init->HasSideEffects(CGM.getContext())) return false; continue; @@ -761,15 +779,9 @@ bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) { AllowOverwrite = true; } else { // Otherwise we have a bitfield. - if (auto *CI = dyn_cast<llvm::ConstantInt>(EltInit)) { - if (!AppendBitField(Field, Layout.getFieldOffset(FieldNo), CI, - AllowOverwrite)) - return false; - } else { - // We are trying to initialize a bitfield with a non-trivial constant, - // this must require run-time code. + if (!AppendBitField(Field, Layout.getFieldOffset(FieldNo), EltInit, + AllowOverwrite)) return false; - } } } @@ -800,8 +812,14 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, // Add a vtable pointer, if we need one and it hasn't already been added. if (Layout.hasOwnVFPtr()) { llvm::Constant *VTableAddressPoint = - CGM.getCXXABI().getVTableAddressPointForConstExpr( - BaseSubobject(CD, Offset), VTableClass); + CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset), + VTableClass); + if (auto Authentication = CGM.getVTablePointerAuthentication(CD)) { + VTableAddressPoint = Emitter.tryEmitConstantSignedPointer( + VTableAddressPoint, *Authentication); + if (!VTableAddressPoint) + return false; + } if (!AppendBytes(Offset, VTableAddressPoint)) return false; } @@ -840,7 +858,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, continue; // Don't emit anonymous bitfields or zero-sized fields. - if (Field->isUnnamedBitfield() || Field->isZeroSize(CGM.getContext())) + if (Field->isUnnamedBitField() || + isEmptyFieldForLayout(CGM.getContext(), *Field)) continue; // Emit the value of the initializer. @@ -863,7 +882,7 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, } else { // Otherwise we have a bitfield. if (!AppendBitField(*Field, Layout.getFieldOffset(FieldNo) + OffsetBits, - cast<llvm::ConstantInt>(EltInit), AllowOverwrite)) + EltInit, AllowOverwrite)) return false; } } @@ -879,7 +898,7 @@ llvm::Constant *ConstStructBuilder::Finalize(QualType Type) { } llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter, - InitListExpr *ILE, + const InitListExpr *ILE, QualType ValTy) { ConstantAggregateBuilder Const(Emitter.CGM); ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero()); @@ -906,7 +925,8 @@ llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter, bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter, ConstantAggregateBuilder &Const, - CharUnits Offset, InitListExpr *Updater) { + CharUnits Offset, + const InitListExpr *Updater) { return ConstStructBuilder(Emitter, Const, Offset) .Build(Updater, /*AllowOverwrite*/ true); } @@ -947,11 +967,11 @@ tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter, static llvm::Constant * EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, - llvm::Type *CommonElementType, unsigned ArrayBound, + llvm::Type *CommonElementType, uint64_t ArrayBound, SmallVectorImpl<llvm::Constant *> &Elements, llvm::Constant *Filler) { // Figure out how long the initial prefix of non-zero elements is. - unsigned NonzeroLength = ArrayBound; + uint64_t NonzeroLength = ArrayBound; if (Elements.size() < NonzeroLength && Filler->isNullValue()) NonzeroLength = Elements.size(); if (NonzeroLength == Elements.size()) { @@ -963,7 +983,7 @@ EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, return llvm::ConstantAggregateZero::get(DesiredType); // Add a zeroinitializer array filler if we have lots of trailing zeroes. - unsigned TrailingZeroes = ArrayBound - NonzeroLength; + uint64_t TrailingZeroes = ArrayBound - NonzeroLength; if (TrailingZeroes >= 8) { assert(Elements.size() >= NonzeroLength && "missing initializer for non-zero element"); @@ -1013,8 +1033,8 @@ EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, // // Constant folding is currently missing support for a few features supported // here: CK_ToUnion, CK_ReinterpretMemberPointer, and DesignatedInitUpdateExpr. -class ConstExprEmitter : - public StmtVisitor<ConstExprEmitter, llvm::Constant*, QualType> { +class ConstExprEmitter + : public ConstStmtVisitor<ConstExprEmitter, llvm::Constant *, QualType> { CodeGenModule &CGM; ConstantEmitter &Emitter; llvm::LLVMContext &VMContext; @@ -1027,43 +1047,60 @@ public: // Visitor Methods //===--------------------------------------------------------------------===// - llvm::Constant *VisitStmt(Stmt *S, QualType T) { - return nullptr; - } + llvm::Constant *VisitStmt(const Stmt *S, QualType T) { return nullptr; } - llvm::Constant *VisitConstantExpr(ConstantExpr *CE, QualType T) { + llvm::Constant *VisitConstantExpr(const ConstantExpr *CE, QualType T) { if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE)) return Result; return Visit(CE->getSubExpr(), T); } - llvm::Constant *VisitParenExpr(ParenExpr *PE, QualType T) { + llvm::Constant *VisitParenExpr(const ParenExpr *PE, QualType T) { return Visit(PE->getSubExpr(), T); } llvm::Constant * - VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *PE, + VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *PE, QualType T) { return Visit(PE->getReplacement(), T); } - llvm::Constant *VisitGenericSelectionExpr(GenericSelectionExpr *GE, + llvm::Constant *VisitGenericSelectionExpr(const GenericSelectionExpr *GE, QualType T) { return Visit(GE->getResultExpr(), T); } - llvm::Constant *VisitChooseExpr(ChooseExpr *CE, QualType T) { + llvm::Constant *VisitChooseExpr(const ChooseExpr *CE, QualType T) { return Visit(CE->getChosenSubExpr(), T); } - llvm::Constant *VisitCompoundLiteralExpr(CompoundLiteralExpr *E, QualType T) { + llvm::Constant *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E, + QualType T) { return Visit(E->getInitializer(), T); } - llvm::Constant *VisitCastExpr(CastExpr *E, QualType destType) { + llvm::Constant *ProduceIntToIntCast(const Expr *E, QualType DestType) { + QualType FromType = E->getType(); + // See also HandleIntToIntCast in ExprConstant.cpp + if (FromType->isIntegerType()) + if (llvm::Constant *C = Visit(E, FromType)) + if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) { + unsigned SrcWidth = CGM.getContext().getIntWidth(FromType); + unsigned DstWidth = CGM.getContext().getIntWidth(DestType); + if (DstWidth == SrcWidth) + return CI; + llvm::APInt A = FromType->isSignedIntegerType() + ? CI->getValue().sextOrTrunc(DstWidth) + : CI->getValue().zextOrTrunc(DstWidth); + return llvm::ConstantInt::get(CGM.getLLVMContext(), A); + } + return nullptr; + } + + llvm::Constant *VisitCastExpr(const CastExpr *E, QualType destType) { if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) CGM.EmitExplicitCastExprType(ECE, Emitter.CGF); - Expr *subExpr = E->getSubExpr(); + const Expr *subExpr = E->getSubExpr(); switch (E->getCastKind()) { case CK_ToUnion: { @@ -1117,7 +1154,8 @@ public: // interesting conversions should be done in Evaluate(). But as a // special case, allow compound literals to support the gcc extension // allowing "struct x {int x;} x = (struct x) {};". - if (auto *E = dyn_cast<CompoundLiteralExpr>(subExpr->IgnoreParens())) + if (const auto *E = + dyn_cast<CompoundLiteralExpr>(subExpr->IgnoreParens())) return Visit(E->getInitializer(), destType); return nullptr; } @@ -1140,23 +1178,8 @@ public: case CK_IntToOCLSampler: llvm_unreachable("global sampler variables are not generated"); - case CK_IntegralCast: { - QualType FromType = subExpr->getType(); - // See also HandleIntToIntCast in ExprConstant.cpp - if (FromType->isIntegerType()) - if (llvm::Constant *C = Visit(subExpr, FromType)) - if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) { - unsigned SrcWidth = CGM.getContext().getIntWidth(FromType); - unsigned DstWidth = CGM.getContext().getIntWidth(destType); - if (DstWidth == SrcWidth) - return CI; - llvm::APInt A = FromType->isSignedIntegerType() - ? CI->getValue().sextOrTrunc(DstWidth) - : CI->getValue().zextOrTrunc(DstWidth); - return llvm::ConstantInt::get(CGM.getLLVMContext(), A); - } - return nullptr; - } + case CK_IntegralCast: + return ProduceIntToIntCast(subExpr, destType); case CK_Dependent: llvm_unreachable("saw dependent cast!"); @@ -1225,63 +1248,131 @@ public: case CK_IntegralToFixedPoint: case CK_ZeroToOCLOpaqueType: case CK_MatrixCast: + case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: return nullptr; } llvm_unreachable("Invalid CastKind"); } - llvm::Constant *VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE, QualType T) { + llvm::Constant *VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *DIE, + QualType T) { // No need for a DefaultInitExprScope: we don't handle 'this' in a // constant expression. return Visit(DIE->getExpr(), T); } - llvm::Constant *VisitExprWithCleanups(ExprWithCleanups *E, QualType T) { + llvm::Constant *VisitExprWithCleanups(const ExprWithCleanups *E, QualType T) { return Visit(E->getSubExpr(), T); } - llvm::Constant *VisitIntegerLiteral(IntegerLiteral *I, QualType T) { + llvm::Constant *VisitIntegerLiteral(const IntegerLiteral *I, QualType T) { return llvm::ConstantInt::get(CGM.getLLVMContext(), I->getValue()); } - llvm::Constant *EmitArrayInitialization(InitListExpr *ILE, QualType T) { + static APValue withDestType(ASTContext &Ctx, const Expr *E, QualType SrcType, + QualType DestType, const llvm::APSInt &Value) { + if (!Ctx.hasSameType(SrcType, DestType)) { + if (DestType->isFloatingType()) { + llvm::APFloat Result = + llvm::APFloat(Ctx.getFloatTypeSemantics(DestType), 1); + llvm::RoundingMode RM = + E->getFPFeaturesInEffect(Ctx.getLangOpts()).getRoundingMode(); + if (RM == llvm::RoundingMode::Dynamic) + RM = llvm::RoundingMode::NearestTiesToEven; + Result.convertFromAPInt(Value, Value.isSigned(), RM); + return APValue(Result); + } + } + return APValue(Value); + } + + llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) { auto *CAT = CGM.getContext().getAsConstantArrayType(ILE->getType()); assert(CAT && "can't emit array init for non-constant-bound array"); - unsigned NumInitElements = ILE->getNumInits(); - unsigned NumElements = CAT->getSize().getZExtValue(); + uint64_t NumInitElements = ILE->getNumInits(); + const uint64_t NumElements = CAT->getZExtSize(); + for (const auto *Init : ILE->inits()) { + if (const auto *Embed = + dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) { + NumInitElements += Embed->getDataElementCount() - 1; + if (NumInitElements > NumElements) { + NumInitElements = NumElements; + break; + } + } + } // Initialising an array requires us to automatically // initialise any elements that have not been initialised explicitly - unsigned NumInitableElts = std::min(NumInitElements, NumElements); + uint64_t NumInitableElts = std::min<uint64_t>(NumInitElements, NumElements); QualType EltType = CAT->getElementType(); // Initialize remaining array elements. llvm::Constant *fillC = nullptr; - if (Expr *filler = ILE->getArrayFiller()) { + if (const Expr *filler = ILE->getArrayFiller()) { fillC = Emitter.tryEmitAbstractForMemory(filler, EltType); if (!fillC) return nullptr; } // Copy initializer elements. - SmallVector<llvm::Constant*, 16> Elts; + SmallVector<llvm::Constant *, 16> Elts; if (fillC && fillC->isNullValue()) Elts.reserve(NumInitableElts + 1); else Elts.reserve(NumElements); llvm::Type *CommonElementType = nullptr; - for (unsigned i = 0; i < NumInitableElts; ++i) { - Expr *Init = ILE->getInit(i); - llvm::Constant *C = Emitter.tryEmitPrivateForMemory(Init, EltType); + auto Emit = [&](const Expr *Init, unsigned ArrayIndex) { + llvm::Constant *C = nullptr; + C = Emitter.tryEmitPrivateForMemory(Init, EltType); if (!C) - return nullptr; - if (i == 0) + return false; + if (ArrayIndex == 0) CommonElementType = C->getType(); else if (C->getType() != CommonElementType) CommonElementType = nullptr; Elts.push_back(C); + return true; + }; + + unsigned ArrayIndex = 0; + QualType DestTy = CAT->getElementType(); + for (unsigned i = 0; i < ILE->getNumInits(); ++i) { + const Expr *Init = ILE->getInit(i); + if (auto *EmbedS = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) { + StringLiteral *SL = EmbedS->getDataStringLiteral(); + llvm::APSInt Value(CGM.getContext().getTypeSize(DestTy), + DestTy->isUnsignedIntegerType()); + llvm::Constant *C; + for (unsigned I = EmbedS->getStartingElementPos(), + N = EmbedS->getDataElementCount(); + I != EmbedS->getStartingElementPos() + N; ++I) { + Value = SL->getCodeUnit(I); + if (DestTy->isIntegerType()) { + C = llvm::ConstantInt::get(CGM.getLLVMContext(), Value); + } else { + C = Emitter.tryEmitPrivateForMemory( + withDestType(CGM.getContext(), Init, EmbedS->getType(), DestTy, + Value), + EltType); + } + if (!C) + return nullptr; + Elts.push_back(C); + ArrayIndex++; + } + if ((ArrayIndex - EmbedS->getDataElementCount()) == 0) + CommonElementType = C->getType(); + else if (C->getType() != CommonElementType) + CommonElementType = nullptr; + } else { + if (!Emit(Init, ArrayIndex)) + return nullptr; + ArrayIndex++; + } } llvm::ArrayType *Desired = @@ -1290,16 +1381,17 @@ public: fillC); } - llvm::Constant *EmitRecordInitialization(InitListExpr *ILE, QualType T) { + llvm::Constant *EmitRecordInitialization(const InitListExpr *ILE, + QualType T) { return ConstStructBuilder::BuildStruct(Emitter, ILE, T); } - llvm::Constant *VisitImplicitValueInitExpr(ImplicitValueInitExpr* E, + llvm::Constant *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E, QualType T) { return CGM.EmitNullConstant(T); } - llvm::Constant *VisitInitListExpr(InitListExpr *ILE, QualType T) { + llvm::Constant *VisitInitListExpr(const InitListExpr *ILE, QualType T) { if (ILE->isTransparent()) return Visit(ILE->getInit(0), T); @@ -1312,8 +1404,9 @@ public: return nullptr; } - llvm::Constant *VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E, - QualType destType) { + llvm::Constant * + VisitDesignatedInitUpdateExpr(const DesignatedInitUpdateExpr *E, + QualType destType) { auto C = Visit(E->getBase(), destType); if (!C) return nullptr; @@ -1327,12 +1420,13 @@ public: llvm::Type *ValTy = CGM.getTypes().ConvertType(destType); bool HasFlexibleArray = false; - if (auto *RT = destType->getAs<RecordType>()) + if (const auto *RT = destType->getAs<RecordType>()) HasFlexibleArray = RT->getDecl()->hasFlexibleArrayMember(); return Const.build(ValTy, HasFlexibleArray); } - llvm::Constant *VisitCXXConstructExpr(CXXConstructExpr *E, QualType Ty) { + llvm::Constant *VisitCXXConstructExpr(const CXXConstructExpr *E, + QualType Ty) { if (!E->getConstructor()->isTrivial()) return nullptr; @@ -1342,13 +1436,13 @@ public: assert(E->getConstructor()->isCopyOrMoveConstructor() && "trivial ctor has argument but isn't a copy/move ctor"); - Expr *Arg = E->getArg(0); + const Expr *Arg = E->getArg(0); assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) && "argument to copy ctor is of wrong type"); // Look through the temporary; it's just converting the value to an // lvalue to pass it to the constructor. - if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Arg)) + if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Arg)) return Visit(MTE->getSubExpr(), Ty); // Don't try to support arbitrary lvalue-to-rvalue conversions for now. return nullptr; @@ -1357,12 +1451,12 @@ public: return CGM.EmitNullConstant(Ty); } - llvm::Constant *VisitStringLiteral(StringLiteral *E, QualType T) { + llvm::Constant *VisitStringLiteral(const StringLiteral *E, QualType T) { // This is a string literal initializing an array in an initializer. return CGM.GetConstantArrayFromStringLiteral(E); } - llvm::Constant *VisitObjCEncodeExpr(ObjCEncodeExpr *E, QualType T) { + llvm::Constant *VisitObjCEncodeExpr(const ObjCEncodeExpr *E, QualType T) { // This must be an @encode initializing an array in a static initializer. // Don't emit it as the address of the string, emit the string data itself // as an inline array. @@ -1373,7 +1467,7 @@ public: // Resize the string to the right size, adding zeros at the end, or // truncating as needed. - Str.resize(CAT->getSize().getZExtValue(), '\0'); + Str.resize(CAT->getZExtSize(), '\0'); return llvm::ConstantDataArray::getString(VMContext, Str, false); } @@ -1381,13 +1475,17 @@ public: return Visit(E->getSubExpr(), T); } - llvm::Constant *VisitUnaryMinus(UnaryOperator *U, QualType T) { + llvm::Constant *VisitUnaryMinus(const UnaryOperator *U, QualType T) { if (llvm::Constant *C = Visit(U->getSubExpr(), T)) if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) return llvm::ConstantInt::get(CGM.getLLVMContext(), -CI->getValue()); return nullptr; } + llvm::Constant *VisitPackIndexingExpr(const PackIndexingExpr *E, QualType T) { + return Visit(E->getSelectedExpr(), T); + } + // Utility methods llvm::Type *ConvertType(QualType T) { return CGM.getTypes().ConvertType(T); @@ -1455,9 +1553,11 @@ ConstantEmitter::emitAbstract(const Expr *E, QualType destType) { llvm::Constant * ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value, - QualType destType) { + QualType destType, + bool EnablePtrAuthFunctionTypeDiscrimination) { auto state = pushAbstract(); - auto C = tryEmitPrivate(value, destType); + auto C = + tryEmitPrivate(value, destType, EnablePtrAuthFunctionTypeDiscrimination); C = validateAndPopAbstract(C, state); if (!C) { CGM.Error(loc, @@ -1567,7 +1667,7 @@ namespace { // messing around with llvm::Constant structures, which never itself // does anything that should be visible in compiler output. for (auto &entry : Locations) { - assert(entry.first->getParent() == nullptr && "not a placeholder!"); + assert(entry.first->getName() == "" && "not a placeholder!"); entry.first->replaceAllUsesWith(entry.second); entry.first->eraseFromParent(); } @@ -1690,8 +1790,7 @@ llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) { if (!destType->isReferenceType()) { QualType nonMemoryDestType = getNonMemoryType(CGM, destType); - if (llvm::Constant *C = ConstExprEmitter(*this).Visit(const_cast<Expr *>(E), - nonMemoryDestType)) + if (llvm::Constant *C = ConstExprEmitter(*this).Visit(E, nonMemoryDestType)) return emitForMemory(C, destType); } @@ -1732,6 +1831,43 @@ llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const APValue &value, return (C ? emitForMemory(C, destType) : nullptr); } +/// Try to emit a constant signed pointer, given a raw pointer and the +/// destination ptrauth qualifier. +/// +/// This can fail if the qualifier needs address discrimination and the +/// emitter is in an abstract mode. +llvm::Constant * +ConstantEmitter::tryEmitConstantSignedPointer(llvm::Constant *UnsignedPointer, + PointerAuthQualifier Schema) { + assert(Schema && "applying trivial ptrauth schema"); + + if (Schema.hasKeyNone()) + return UnsignedPointer; + + unsigned Key = Schema.getKey(); + + // Create an address placeholder if we're using address discrimination. + llvm::GlobalValue *StorageAddress = nullptr; + if (Schema.isAddressDiscriminated()) { + // We can't do this if the emitter is in an abstract state. + if (isAbstract()) + return nullptr; + + StorageAddress = getCurrentAddrPrivate(); + } + + llvm::ConstantInt *Discriminator = + llvm::ConstantInt::get(CGM.IntPtrTy, Schema.getExtraDiscriminator()); + + llvm::Constant *SignedPointer = CGM.getConstantSignedPointer( + UnsignedPointer, Key, StorageAddress, Discriminator); + + if (Schema.isAddressDiscriminated()) + registerCurrentAddrPrivate(SignedPointer, StorageAddress); + + return SignedPointer; +} + llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM, llvm::Constant *C, QualType destType) { @@ -1763,6 +1899,27 @@ llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM, return Res; } + if (destType->isBitIntType()) { + ConstantAggregateBuilder Builder(CGM); + llvm::Type *LoadStoreTy = CGM.getTypes().convertTypeForLoadStore(destType); + // ptrtoint/inttoptr should not involve _BitInt in constant expressions, so + // casting to ConstantInt is safe here. + auto *CI = cast<llvm::ConstantInt>(C); + llvm::Constant *Res = llvm::ConstantFoldCastOperand( + destType->isSignedIntegerOrEnumerationType() ? llvm::Instruction::SExt + : llvm::Instruction::ZExt, + CI, LoadStoreTy, CGM.getDataLayout()); + if (CGM.getTypes().typeRequiresSplitIntoByteArray(destType, C->getType())) { + // Long _BitInt has array of bytes as in-memory type. + // So, split constant into individual bytes. + llvm::Type *DesiredTy = CGM.getTypes().ConvertTypeForMem(destType); + llvm::APInt Value = cast<llvm::ConstantInt>(Res)->getValue(); + Builder.addBits(Value, /*OffsetInBits=*/0, /*AllowOverwrite=*/false); + return Builder.build(DesiredTy, /*AllowOversized*/ false); + } + return Res; + } + return C; } @@ -1771,8 +1928,7 @@ llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E, assert(!destType->isVoidType() && "can't emit a void constant"); if (!destType->isReferenceType()) - if (llvm::Constant *C = - ConstExprEmitter(*this).Visit(const_cast<Expr *>(E), destType)) + if (llvm::Constant *C = ConstExprEmitter(*this).Visit(E, destType)) return C; Expr::EvalResult Result; @@ -1816,14 +1972,18 @@ class ConstantLValueEmitter : public ConstStmtVisitor<ConstantLValueEmitter, ConstantEmitter &Emitter; const APValue &Value; QualType DestType; + bool EnablePtrAuthFunctionTypeDiscrimination; // Befriend StmtVisitorBase so that we don't have to expose Visit*. friend StmtVisitorBase; public: ConstantLValueEmitter(ConstantEmitter &emitter, const APValue &value, - QualType destType) - : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType) {} + QualType destType, + bool EnablePtrAuthFunctionTypeDiscrimination = true) + : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType), + EnablePtrAuthFunctionTypeDiscrimination( + EnablePtrAuthFunctionTypeDiscrimination) {} llvm::Constant *tryEmit(); @@ -1846,6 +2006,12 @@ private: ConstantLValue VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *E); + ConstantLValue emitPointerAuthSignConstant(const CallExpr *E); + llvm::Constant *emitPointerAuthPointer(const Expr *E); + unsigned emitPointerAuthKey(const Expr *E); + std::pair<llvm::Constant *, llvm::ConstantInt *> + emitPointerAuthDiscriminator(const Expr *E); + bool hasNonZeroOffset() const { return !Value.getLValueOffset().isZero(); } @@ -1940,10 +2106,30 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) { if (D->hasAttr<WeakRefAttr>()) return CGM.GetWeakRefReference(D).getPointer(); - if (auto FD = dyn_cast<FunctionDecl>(D)) - return CGM.GetAddrOfFunction(FD); + auto PtrAuthSign = [&](llvm::Constant *C) { + CGPointerAuthInfo AuthInfo; + + if (EnablePtrAuthFunctionTypeDiscrimination) + AuthInfo = CGM.getFunctionPointerAuthInfo(DestType); - if (auto VD = dyn_cast<VarDecl>(D)) { + if (AuthInfo) { + if (hasNonZeroOffset()) + return ConstantLValue(nullptr); + + C = applyOffset(C); + C = CGM.getConstantSignedPointer( + C, AuthInfo.getKey(), nullptr, + cast_or_null<llvm::ConstantInt>(AuthInfo.getDiscriminator())); + return ConstantLValue(C, /*applied offset*/ true); + } + + return ConstantLValue(C); + }; + + if (const auto *FD = dyn_cast<FunctionDecl>(D)) + return PtrAuthSign(CGM.getRawFunctionPointer(FD)); + + if (const auto *VD = dyn_cast<VarDecl>(D)) { // We can never refer to a variable with local storage. if (!VD->hasLocalStorage()) { if (VD->isFileVarDecl() || VD->hasExternalStorage()) @@ -1956,13 +2142,13 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) { } } - if (auto *GD = dyn_cast<MSGuidDecl>(D)) + if (const auto *GD = dyn_cast<MSGuidDecl>(D)) return CGM.GetAddrOfMSGuidDecl(GD); - if (auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(D)) + if (const auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(D)) return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD); - if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(D)) + if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(D)) return CGM.GetAddrOfTemplateParamObject(TPO); return nullptr; @@ -2016,7 +2202,7 @@ ConstantLValue ConstantLValueEmitter::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) { assert(E->isExpressibleAsConstantInitializer() && "this boxed expression can't be emitted as a compile-time constant"); - auto *SL = cast<StringLiteral>(E->getSubExpr()->IgnoreParenCasts()); + const auto *SL = cast<StringLiteral>(E->getSubExpr()->IgnoreParenCasts()); return emitConstantObjCStringLiteral(SL, E->getType(), CGM); } @@ -2038,20 +2224,75 @@ ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) { if (builtin == Builtin::BI__builtin_function_start) return CGM.GetFunctionStart( E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext())); + + if (builtin == Builtin::BI__builtin_ptrauth_sign_constant) + return emitPointerAuthSignConstant(E); + if (builtin != Builtin::BI__builtin___CFStringMakeConstantString && builtin != Builtin::BI__builtin___NSStringMakeConstantString) return nullptr; - auto literal = cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts()); + const auto *Literal = cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts()); if (builtin == Builtin::BI__builtin___NSStringMakeConstantString) { - return CGM.getObjCRuntime().GenerateConstantString(literal); + return CGM.getObjCRuntime().GenerateConstantString(Literal); } else { // FIXME: need to deal with UCN conversion issues. - return CGM.GetAddrOfConstantCFString(literal); + return CGM.GetAddrOfConstantCFString(Literal); } } ConstantLValue +ConstantLValueEmitter::emitPointerAuthSignConstant(const CallExpr *E) { + llvm::Constant *UnsignedPointer = emitPointerAuthPointer(E->getArg(0)); + unsigned Key = emitPointerAuthKey(E->getArg(1)); + auto [StorageAddress, OtherDiscriminator] = + emitPointerAuthDiscriminator(E->getArg(2)); + + llvm::Constant *SignedPointer = CGM.getConstantSignedPointer( + UnsignedPointer, Key, StorageAddress, OtherDiscriminator); + return SignedPointer; +} + +llvm::Constant *ConstantLValueEmitter::emitPointerAuthPointer(const Expr *E) { + Expr::EvalResult Result; + bool Succeeded = E->EvaluateAsRValue(Result, CGM.getContext()); + assert(Succeeded); + (void)Succeeded; + + // The assertions here are all checked by Sema. + assert(Result.Val.isLValue()); + if (isa<FunctionDecl>(Result.Val.getLValueBase().get<const ValueDecl *>())) + assert(Result.Val.getLValueOffset().isZero()); + return ConstantEmitter(CGM, Emitter.CGF) + .emitAbstract(E->getExprLoc(), Result.Val, E->getType(), false); +} + +unsigned ConstantLValueEmitter::emitPointerAuthKey(const Expr *E) { + return E->EvaluateKnownConstInt(CGM.getContext()).getZExtValue(); +} + +std::pair<llvm::Constant *, llvm::ConstantInt *> +ConstantLValueEmitter::emitPointerAuthDiscriminator(const Expr *E) { + E = E->IgnoreParens(); + + if (const auto *Call = dyn_cast<CallExpr>(E)) { + if (Call->getBuiltinCallee() == + Builtin::BI__builtin_ptrauth_blend_discriminator) { + llvm::Constant *Pointer = ConstantEmitter(CGM).emitAbstract( + Call->getArg(0), Call->getArg(0)->getType()); + auto *Extra = cast<llvm::ConstantInt>(ConstantEmitter(CGM).emitAbstract( + Call->getArg(1), Call->getArg(1)->getType())); + return {Pointer, Extra}; + } + } + + llvm::Constant *Result = ConstantEmitter(CGM).emitAbstract(E, E->getType()); + if (Result->getType()->isPointerTy()) + return {Result, nullptr}; + return {nullptr, cast<llvm::ConstantInt>(Result)}; +} + +ConstantLValue ConstantLValueEmitter::VisitBlockExpr(const BlockExpr *E) { StringRef functionName; if (auto CGF = Emitter.CGF) @@ -2076,22 +2317,22 @@ ConstantLValue ConstantLValueEmitter::VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *E) { assert(E->getStorageDuration() == SD_Static); - SmallVector<const Expr *, 2> CommaLHSs; - SmallVector<SubobjectAdjustment, 2> Adjustments; - const Expr *Inner = - E->getSubExpr()->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments); + const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments(); return CGM.GetAddrOfGlobalTemporary(E, Inner); } -llvm::Constant *ConstantEmitter::tryEmitPrivate(const APValue &Value, - QualType DestType) { +llvm::Constant * +ConstantEmitter::tryEmitPrivate(const APValue &Value, QualType DestType, + bool EnablePtrAuthFunctionTypeDiscrimination) { switch (Value.getKind()) { case APValue::None: case APValue::Indeterminate: // Out-of-lifetime and indeterminate values can be modeled as 'undef'. return llvm::UndefValue::get(CGM.getTypes().ConvertType(DestType)); case APValue::LValue: - return ConstantLValueEmitter(*this, Value, DestType).tryEmit(); + return ConstantLValueEmitter(*this, Value, DestType, + EnablePtrAuthFunctionTypeDiscrimination) + .tryEmit(); case APValue::Int: return llvm::ConstantInt::get(CGM.getLLVMContext(), Value.getInt()); case APValue::FixedPoint: @@ -2286,8 +2527,10 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); // Ignore empty bases. - if (base->isEmpty() || - CGM.getContext().getASTRecordLayout(base).getNonVirtualSize() + if (isEmptyRecordForLayout(CGM.getContext(), I.getType()) || + CGM.getContext() + .getASTRecordLayout(base) + .getNonVirtualSize() .isZero()) continue; @@ -2301,7 +2544,8 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, for (const auto *Field : record->fields()) { // Fill in non-bitfields. (Bitfields always use a zero pattern, which we // will fill in later.) - if (!Field->isBitField() && !Field->isZeroSize(CGM.getContext())) { + if (!Field->isBitField() && + !isEmptyFieldForLayout(CGM.getContext(), Field)) { unsigned fieldIndex = layout.getLLVMFieldNo(Field); elements[fieldIndex] = CGM.EmitNullConstant(Field->getType()); } @@ -2323,7 +2567,7 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); // Ignore empty bases. - if (base->isEmpty()) + if (isEmptyRecordForLayout(CGM.getContext(), I.getType())) continue; unsigned fieldIndex = layout.getVirtualBaseIndex(base); @@ -2380,7 +2624,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { llvm::Constant *Element = ConstantEmitter::emitNullForMemory(*this, ElementTy); - unsigned NumElements = CAT->getSize().getZExtValue(); + unsigned NumElements = CAT->getZExtSize(); SmallVector<llvm::Constant *, 8> Array(NumElements, Element); return llvm::ConstantArray::get(ATy, Array); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp index 181b15e9c7d0..6e212e74676e 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp @@ -15,6 +15,7 @@ #include "CGDebugInfo.h" #include "CGObjCRuntime.h" #include "CGOpenMPRuntime.h" +#include "CGRecordLayout.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "ConstantEmitter.h" @@ -52,6 +53,10 @@ using llvm::Value; // Scalar Expression Emitter //===----------------------------------------------------------------------===// +namespace llvm { +extern cl::opt<bool> EnableSingleByteCoverage; +} // namespace llvm + namespace { /// Determine whether the given binary operation may overflow. @@ -142,6 +147,15 @@ struct BinOpInfo { return UnOp->getSubExpr()->getType()->isFixedPointType(); return false; } + + /// Check if the RHS has a signed integer representation. + bool rhsHasSignedIntegerRepresentation() const { + if (const auto *BinOp = dyn_cast<BinaryOperator>(E)) { + QualType RHSType = BinOp->getRHS()->getType(); + return RHSType->hasSignedIntegerRepresentation(); + } + return false; + } }; static bool MustVisitNullValue(const Expr *E) { @@ -304,6 +318,7 @@ public: llvm::Type *DstTy, SourceLocation Loc); /// Known implicit conversion check kinds. + /// This is used for bitfield conversion checks as well. /// Keep in sync with the enum of the same name in ubsan_handlers.h enum ImplicitConversionCheckKind : unsigned char { ICCK_IntegerTruncation = 0, // Legacy, was only used by clang 7. @@ -421,9 +436,10 @@ public: if (Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) { if (E->isGLValue()) - return CGF.Builder.CreateLoad(Address( - Result, CGF.ConvertTypeForMem(E->getType()), - CGF.getContext().getTypeAlignInChars(E->getType()))); + return CGF.EmitLoadOfScalar( + Address(Result, CGF.convertTypeForLoadStore(E->getType()), + CGF.getContext().getTypeAlignInChars(E->getType())), + /*Volatile*/ false, E->getType(), E->getExprLoc()); return Result; } return Visit(E->getSubExpr()); @@ -491,6 +507,7 @@ public: } Value *VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E); + Value *VisitEmbedExpr(EmbedExpr *E); Value *VisitOpaqueValueExpr(OpaqueValueExpr *E) { if (E->isGLValue()) @@ -723,7 +740,9 @@ public: if (Ops.Ty->isSignedIntegerOrEnumerationType()) { switch (CGF.getLangOpts().getSignedOverflowBehavior()) { case LangOptions::SOB_Defined: - return Builder.CreateMul(Ops.LHS, Ops.RHS, "mul"); + if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) + return Builder.CreateMul(Ops.LHS, Ops.RHS, "mul"); + [[fallthrough]]; case LangOptions::SOB_Undefined: if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) return Builder.CreateNSWMul(Ops.LHS, Ops.RHS, "mul"); @@ -774,7 +793,7 @@ public: void EmitUndefinedBehaviorIntegerDivAndRemCheck(const BinOpInfo &Ops, llvm::Value *Zero,bool isDiv); // Common helper for getting how wide LHS of shift is. - static Value *GetWidthMinusOneValue(Value* LHS,Value* RHS); + static Value *GetMaximumShiftAmount(Value *LHS, Value *RHS, bool RHSIsSigned); // Used for shifting constraints for OpenCL, do mask for powers of 2, URem for // non powers of two. @@ -904,6 +923,9 @@ public: } Value *VisitAsTypeExpr(AsTypeExpr *CE); Value *VisitAtomicExpr(AtomicExpr *AE); + Value *VisitPackIndexingExpr(PackIndexingExpr *E) { + return Visit(E->getSelectedExpr()); + } }; } // end anonymous namespace. @@ -1089,11 +1111,28 @@ void ScalarExprEmitter::EmitIntegerTruncationCheck(Value *Src, QualType SrcType, llvm::Constant *StaticArgs[] = { CGF.EmitCheckSourceLocation(Loc), CGF.EmitCheckTypeDescriptor(SrcType), CGF.EmitCheckTypeDescriptor(DstType), - llvm::ConstantInt::get(Builder.getInt8Ty(), Check.first)}; + llvm::ConstantInt::get(Builder.getInt8Ty(), Check.first), + llvm::ConstantInt::get(Builder.getInt32Ty(), 0)}; + CGF.EmitCheck(Check.second, SanitizerHandler::ImplicitConversion, StaticArgs, {Src, Dst}); } +static llvm::Value *EmitIsNegativeTestHelper(Value *V, QualType VType, + const char *Name, + CGBuilderTy &Builder) { + bool VSigned = VType->isSignedIntegerOrEnumerationType(); + llvm::Type *VTy = V->getType(); + if (!VSigned) { + // If the value is unsigned, then it is never negative. + return llvm::ConstantInt::getFalse(VTy->getContext()); + } + llvm::Constant *Zero = llvm::ConstantInt::get(VTy, 0); + return Builder.CreateICmp(llvm::ICmpInst::ICMP_SLT, V, Zero, + llvm::Twine(Name) + "." + V->getName() + + ".negativitycheck"); +} + // Should be called within CodeGenFunction::SanitizerScope RAII scope. // Returns 'i1 false' when the conversion Src -> Dst changed the sign. static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind, @@ -1118,30 +1157,12 @@ EmitIntegerSignChangeCheckHelper(Value *Src, QualType SrcType, Value *Dst, assert(((SrcBits != DstBits) || (SrcSigned != DstSigned)) && "either the widths should be different, or the signednesses."); - // NOTE: zero value is considered to be non-negative. - auto EmitIsNegativeTest = [&Builder](Value *V, QualType VType, - const char *Name) -> Value * { - // Is this value a signed type? - bool VSigned = VType->isSignedIntegerOrEnumerationType(); - llvm::Type *VTy = V->getType(); - if (!VSigned) { - // If the value is unsigned, then it is never negative. - // FIXME: can we encounter non-scalar VTy here? - return llvm::ConstantInt::getFalse(VTy->getContext()); - } - // Get the zero of the same type with which we will be comparing. - llvm::Constant *Zero = llvm::ConstantInt::get(VTy, 0); - // %V.isnegative = icmp slt %V, 0 - // I.e is %V *strictly* less than zero, does it have negative value? - return Builder.CreateICmp(llvm::ICmpInst::ICMP_SLT, V, Zero, - llvm::Twine(Name) + "." + V->getName() + - ".negativitycheck"); - }; - // 1. Was the old Value negative? - llvm::Value *SrcIsNegative = EmitIsNegativeTest(Src, SrcType, "src"); + llvm::Value *SrcIsNegative = + EmitIsNegativeTestHelper(Src, SrcType, "src", Builder); // 2. Is the new Value negative? - llvm::Value *DstIsNegative = EmitIsNegativeTest(Dst, DstType, "dst"); + llvm::Value *DstIsNegative = + EmitIsNegativeTestHelper(Dst, DstType, "dst", Builder); // 3. Now, was the 'negativity status' preserved during the conversion? // NOTE: conversion from negative to zero is considered to change the sign. // (We want to get 'false' when the conversion changed the sign) @@ -1230,12 +1251,143 @@ void ScalarExprEmitter::EmitIntegerSignChangeCheck(Value *Src, QualType SrcType, llvm::Constant *StaticArgs[] = { CGF.EmitCheckSourceLocation(Loc), CGF.EmitCheckTypeDescriptor(SrcType), CGF.EmitCheckTypeDescriptor(DstType), - llvm::ConstantInt::get(Builder.getInt8Ty(), CheckKind)}; + llvm::ConstantInt::get(Builder.getInt8Ty(), CheckKind), + llvm::ConstantInt::get(Builder.getInt32Ty(), 0)}; // EmitCheck() will 'and' all the checks together. CGF.EmitCheck(Checks, SanitizerHandler::ImplicitConversion, StaticArgs, {Src, Dst}); } +// Should be called within CodeGenFunction::SanitizerScope RAII scope. +// Returns 'i1 false' when the truncation Src -> Dst was lossy. +static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind, + std::pair<llvm::Value *, SanitizerMask>> +EmitBitfieldTruncationCheckHelper(Value *Src, QualType SrcType, Value *Dst, + QualType DstType, CGBuilderTy &Builder) { + bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType(); + bool DstSigned = DstType->isSignedIntegerOrEnumerationType(); + + ScalarExprEmitter::ImplicitConversionCheckKind Kind; + if (!SrcSigned && !DstSigned) + Kind = ScalarExprEmitter::ICCK_UnsignedIntegerTruncation; + else + Kind = ScalarExprEmitter::ICCK_SignedIntegerTruncation; + + llvm::Value *Check = nullptr; + // 1. Extend the truncated value back to the same width as the Src. + Check = Builder.CreateIntCast(Dst, Src->getType(), DstSigned, "bf.anyext"); + // 2. Equality-compare with the original source value + Check = Builder.CreateICmpEQ(Check, Src, "bf.truncheck"); + // If the comparison result is 'i1 false', then the truncation was lossy. + + return std::make_pair( + Kind, std::make_pair(Check, SanitizerKind::ImplicitBitfieldConversion)); +} + +// Should be called within CodeGenFunction::SanitizerScope RAII scope. +// Returns 'i1 false' when the conversion Src -> Dst changed the sign. +static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind, + std::pair<llvm::Value *, SanitizerMask>> +EmitBitfieldSignChangeCheckHelper(Value *Src, QualType SrcType, Value *Dst, + QualType DstType, CGBuilderTy &Builder) { + // 1. Was the old Value negative? + llvm::Value *SrcIsNegative = + EmitIsNegativeTestHelper(Src, SrcType, "bf.src", Builder); + // 2. Is the new Value negative? + llvm::Value *DstIsNegative = + EmitIsNegativeTestHelper(Dst, DstType, "bf.dst", Builder); + // 3. Now, was the 'negativity status' preserved during the conversion? + // NOTE: conversion from negative to zero is considered to change the sign. + // (We want to get 'false' when the conversion changed the sign) + // So we should just equality-compare the negativity statuses. + llvm::Value *Check = nullptr; + Check = + Builder.CreateICmpEQ(SrcIsNegative, DstIsNegative, "bf.signchangecheck"); + // If the comparison result is 'false', then the conversion changed the sign. + return std::make_pair( + ScalarExprEmitter::ICCK_IntegerSignChange, + std::make_pair(Check, SanitizerKind::ImplicitBitfieldConversion)); +} + +void CodeGenFunction::EmitBitfieldConversionCheck(Value *Src, QualType SrcType, + Value *Dst, QualType DstType, + const CGBitFieldInfo &Info, + SourceLocation Loc) { + + if (!SanOpts.has(SanitizerKind::ImplicitBitfieldConversion)) + return; + + // We only care about int->int conversions here. + // We ignore conversions to/from pointer and/or bool. + if (!PromotionIsPotentiallyEligibleForImplicitIntegerConversionCheck(SrcType, + DstType)) + return; + + if (DstType->isBooleanType() || SrcType->isBooleanType()) + return; + + // This should be truncation of integral types. + assert(isa<llvm::IntegerType>(Src->getType()) && + isa<llvm::IntegerType>(Dst->getType()) && "non-integer llvm type"); + + // TODO: Calculate src width to avoid emitting code + // for unecessary cases. + unsigned SrcBits = ConvertType(SrcType)->getScalarSizeInBits(); + unsigned DstBits = Info.Size; + + bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType(); + bool DstSigned = DstType->isSignedIntegerOrEnumerationType(); + + CodeGenFunction::SanitizerScope SanScope(this); + + std::pair<ScalarExprEmitter::ImplicitConversionCheckKind, + std::pair<llvm::Value *, SanitizerMask>> + Check; + + // Truncation + bool EmitTruncation = DstBits < SrcBits; + // If Dst is signed and Src unsigned, we want to be more specific + // about the CheckKind we emit, in this case we want to emit + // ICCK_SignedIntegerTruncationOrSignChange. + bool EmitTruncationFromUnsignedToSigned = + EmitTruncation && DstSigned && !SrcSigned; + // Sign change + bool SameTypeSameSize = SrcSigned == DstSigned && SrcBits == DstBits; + bool BothUnsigned = !SrcSigned && !DstSigned; + bool LargerSigned = (DstBits > SrcBits) && DstSigned; + // We can avoid emitting sign change checks in some obvious cases + // 1. If Src and Dst have the same signedness and size + // 2. If both are unsigned sign check is unecessary! + // 3. If Dst is signed and bigger than Src, either + // sign-extension or zero-extension will make sure + // the sign remains. + bool EmitSignChange = !SameTypeSameSize && !BothUnsigned && !LargerSigned; + + if (EmitTruncation) + Check = + EmitBitfieldTruncationCheckHelper(Src, SrcType, Dst, DstType, Builder); + else if (EmitSignChange) { + assert(((SrcBits != DstBits) || (SrcSigned != DstSigned)) && + "either the widths should be different, or the signednesses."); + Check = + EmitBitfieldSignChangeCheckHelper(Src, SrcType, Dst, DstType, Builder); + } else + return; + + ScalarExprEmitter::ImplicitConversionCheckKind CheckKind = Check.first; + if (EmitTruncationFromUnsignedToSigned) + CheckKind = ScalarExprEmitter::ICCK_SignedIntegerTruncationOrSignChange; + + llvm::Constant *StaticArgs[] = { + EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(SrcType), + EmitCheckTypeDescriptor(DstType), + llvm::ConstantInt::get(Builder.getInt8Ty(), CheckKind), + llvm::ConstantInt::get(Builder.getInt32Ty(), Info.Size)}; + + EmitCheck(Check.second, SanitizerHandler::ImplicitConversion, StaticArgs, + {Src, Dst}); +} + Value *ScalarExprEmitter::EmitScalarCast(Value *Src, QualType SrcType, QualType DstType, llvm::Type *SrcTy, llvm::Type *DstTy, @@ -1390,7 +1542,7 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, if (auto DstPT = dyn_cast<llvm::PointerType>(DstTy)) { // The source value may be an integer, or a pointer. if (isa<llvm::PointerType>(SrcTy)) - return Builder.CreateBitCast(Src, DstTy, "conv"); + return Src; assert(SrcType->isIntegerType() && "Not ptr->ptr or int->ptr conversion?"); // First, convert to the correct width so that we control the kind of @@ -1646,6 +1798,12 @@ ScalarExprEmitter::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E) { "usn_addr_cast"); } +Value *ScalarExprEmitter::VisitEmbedExpr(EmbedExpr *E) { + assert(E->getDataElementCount() == 1); + auto It = E->begin(); + return Builder.getInt((*It)->getValue()); +} + Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) { // Vector Mask Case if (E->getNumSubExprs() == 2) { @@ -1787,7 +1945,26 @@ Value *ScalarExprEmitter::VisitMemberExpr(MemberExpr *E) { } } - return EmitLoadOfLValue(E); + llvm::Value *Result = EmitLoadOfLValue(E); + + // If -fdebug-info-for-profiling is specified, emit a pseudo variable and its + // debug info for the pointer, even if there is no variable associated with + // the pointer's expression. + if (CGF.CGM.getCodeGenOpts().DebugInfoForProfiling && CGF.getDebugInfo()) { + if (llvm::LoadInst *Load = dyn_cast<llvm::LoadInst>(Result)) { + if (llvm::GetElementPtrInst *GEP = + dyn_cast<llvm::GetElementPtrInst>(Load->getPointerOperand())) { + if (llvm::Instruction *Pointer = + dyn_cast<llvm::Instruction>(GEP->getPointerOperand())) { + QualType Ty = E->getBase()->getType(); + if (!E->isArrow()) + Ty = CGF.getContext().getPointerType(Ty); + CGF.getDebugInfo()->EmitPseudoVariable(Builder, Pointer, Ty); + } + } + } + } + return Result; } Value *ScalarExprEmitter::VisitArraySubscriptExpr(ArraySubscriptExpr *E) { @@ -2062,7 +2239,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_LValueBitCast: case CK_ObjCObjectLValueCast: { - Address Addr = EmitLValue(E).getAddress(CGF); + Address Addr = EmitLValue(E).getAddress(); Addr = Addr.withElementType(CGF.ConvertTypeForMem(DestTy)); LValue LV = CGF.MakeAddrLValue(Addr, DestTy); return EmitLoadOfLValue(LV, CE->getExprLoc()); @@ -2070,8 +2247,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_LValueToRValueBitCast: { LValue SourceLVal = CGF.EmitLValue(E); - Address Addr = SourceLVal.getAddress(CGF).withElementType( - CGF.ConvertTypeForMem(DestTy)); + Address Addr = + SourceLVal.getAddress().withElementType(CGF.ConvertTypeForMem(DestTy)); LValue DestLV = CGF.MakeAddrLValue(Addr, DestTy); DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo()); return EmitLoadOfLValue(DestLV, CE->getExprLoc()); @@ -2134,26 +2311,24 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // If Src is a fixed vector and Dst is a scalable vector, and both have the // same element type, use the llvm.vector.insert intrinsic to perform the // bitcast. - if (const auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) { - if (const auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(DstTy)) { - // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate + if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(SrcTy)) { + if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(DstTy)) { + // If we are casting a fixed i8 vector to a scalable i1 predicate // vector, use a vector insert and bitcast the result. - bool NeedsBitCast = false; - auto PredType = llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16); - llvm::Type *OrigType = DstTy; - if (ScalableDst == PredType && - FixedSrc->getElementType() == Builder.getInt8Ty()) { - DstTy = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2); - ScalableDst = cast<llvm::ScalableVectorType>(DstTy); - NeedsBitCast = true; + if (ScalableDstTy->getElementType()->isIntegerTy(1) && + ScalableDstTy->getElementCount().isKnownMultipleOf(8) && + FixedSrcTy->getElementType()->isIntegerTy(8)) { + ScalableDstTy = llvm::ScalableVectorType::get( + FixedSrcTy->getElementType(), + ScalableDstTy->getElementCount().getKnownMinValue() / 8); } - if (FixedSrc->getElementType() == ScalableDst->getElementType()) { - llvm::Value *UndefVec = llvm::UndefValue::get(DstTy); + if (FixedSrcTy->getElementType() == ScalableDstTy->getElementType()) { + llvm::Value *UndefVec = llvm::UndefValue::get(ScalableDstTy); llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty); llvm::Value *Result = Builder.CreateInsertVector( - DstTy, UndefVec, Src, Zero, "cast.scalable"); - if (NeedsBitCast) - Result = Builder.CreateBitCast(Result, OrigType); + ScalableDstTy, UndefVec, Src, Zero, "cast.scalable"); + if (Result->getType() != DstTy) + Result = Builder.CreateBitCast(Result, DstTy); return Result; } } @@ -2162,18 +2337,19 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // If Src is a scalable vector and Dst is a fixed vector, and both have the // same element type, use the llvm.vector.extract intrinsic to perform the // bitcast. - if (const auto *ScalableSrc = dyn_cast<llvm::ScalableVectorType>(SrcTy)) { - if (const auto *FixedDst = dyn_cast<llvm::FixedVectorType>(DstTy)) { - // If we are casting a scalable 16 x i1 predicate vector to a fixed i8 + if (auto *ScalableSrcTy = dyn_cast<llvm::ScalableVectorType>(SrcTy)) { + if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(DstTy)) { + // If we are casting a scalable i1 predicate vector to a fixed i8 // vector, bitcast the source and use a vector extract. - auto PredType = llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16); - if (ScalableSrc == PredType && - FixedDst->getElementType() == Builder.getInt8Ty()) { - SrcTy = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2); - ScalableSrc = cast<llvm::ScalableVectorType>(SrcTy); - Src = Builder.CreateBitCast(Src, SrcTy); + if (ScalableSrcTy->getElementType()->isIntegerTy(1) && + ScalableSrcTy->getElementCount().isKnownMultipleOf(8) && + FixedDstTy->getElementType()->isIntegerTy(8)) { + ScalableSrcTy = llvm::ScalableVectorType::get( + FixedDstTy->getElementType(), + ScalableSrcTy->getElementCount().getKnownMinValue() / 8); + Src = Builder.CreateBitCast(Src, ScalableSrcTy); } - if (ScalableSrc->getElementType() == FixedDst->getElementType()) { + if (ScalableSrcTy->getElementType() == FixedDstTy->getElementType()) { llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty); return Builder.CreateExtractVector(DstTy, Src, Zero, "cast.fixed"); } @@ -2181,7 +2357,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { } // Perform VLAT <-> VLST bitcast through memory. - // TODO: since the llvm.experimental.vector.{insert,extract} intrinsics + // TODO: since the llvm.vector.{insert,extract} intrinsics // require the element types of the vectors to be the same, we // need to keep this around for bitcasts between VLAT <-> VLST where // the element types of the vectors are not the same, until we figure @@ -2198,7 +2374,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo()); return EmitLoadOfLValue(DestLV, CE->getExprLoc()); } - return Builder.CreateBitCast(Src, DstTy); + + llvm::Value *Result = Builder.CreateBitCast(Src, DstTy); + return CGF.authPointerToPointerCast(Result, E->getType(), DestTy); } case CK_AddressSpaceConversion: { Expr::EvalResult Result; @@ -2242,7 +2420,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // performed and the object is not of the derived type. if (CGF.sanitizePerformTypeCheck()) CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), - Derived.getPointer(), DestTy->getPointeeType()); + Derived, DestTy->getPointeeType()); if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, @@ -2250,13 +2428,14 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CodeGenFunction::CFITCK_DerivedCast, CE->getBeginLoc()); - return Derived.getPointer(); + return CGF.getAsNaturalPointerTo(Derived, CE->getType()->getPointeeType()); } case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { // The EmitPointerWithAlignment path does this fine; just discard // the alignment. - return CGF.EmitPointerWithAlignment(CE).getPointer(); + return CGF.getAsNaturalPointerTo(CGF.EmitPointerWithAlignment(CE), + CE->getType()->getPointeeType()); } case CK_Dynamic: { @@ -2266,7 +2445,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { } case CK_ArrayToPointerDecay: - return CGF.EmitArrayToPointerDecay(E).getPointer(); + return CGF.getAsNaturalPointerTo(CGF.EmitArrayToPointerDecay(E), + CE->getType()->getPointeeType()); case CK_FunctionToPointerDecay: return EmitLValue(E).getPointer(CGF); @@ -2319,6 +2499,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_FloatingComplexToIntegralComplex: case CK_ConstructorConversion: case CK_ToUnion: + case CK_HLSLArrayRValue: llvm_unreachable("scalar cast to non-scalar value"); case CK_LValueToRValue: @@ -2345,6 +2526,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { if (DestTy.mayBeDynamicClass()) IntToPtr = Builder.CreateLaunderInvariantGroup(IntToPtr); } + + IntToPtr = CGF.authPointerToPointerCast(IntToPtr, E->getType(), DestTy); return IntToPtr; } case CK_PointerToIntegral: { @@ -2360,6 +2543,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { PtrExpr = Builder.CreateStripInvariantGroup(PtrExpr); } + PtrExpr = CGF.authPointerToPointerCast(PtrExpr, E->getType(), DestTy); return Builder.CreatePtrToInt(PtrExpr, ConvertType(DestTy)); } case CK_ToVoid: { @@ -2406,6 +2590,12 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CE->getExprLoc()); case CK_IntegralCast: { + if (E->getType()->isExtVectorType() && DestTy->isExtVectorType()) { + QualType SrcElTy = E->getType()->castAs<VectorType>()->getElementType(); + return Builder.CreateIntCast(Visit(E), ConvertType(DestTy), + SrcElTy->isSignedIntegerOrEnumerationType(), + "conv"); + } ScalarConversionOpts Opts; if (auto *ICE = dyn_cast<ImplicitCastExpr>(CE)) { if (!ICE->isPartOfExplicitCast()) @@ -2414,9 +2604,44 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { return EmitScalarConversion(Visit(E), E->getType(), DestTy, CE->getExprLoc(), Opts); } - case CK_IntegralToFloating: - case CK_FloatingToIntegral: - case CK_FloatingCast: + case CK_IntegralToFloating: { + if (E->getType()->isVectorType() && DestTy->isVectorType()) { + // TODO: Support constrained FP intrinsics. + QualType SrcElTy = E->getType()->castAs<VectorType>()->getElementType(); + if (SrcElTy->isSignedIntegerOrEnumerationType()) + return Builder.CreateSIToFP(Visit(E), ConvertType(DestTy), "conv"); + return Builder.CreateUIToFP(Visit(E), ConvertType(DestTy), "conv"); + } + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE); + return EmitScalarConversion(Visit(E), E->getType(), DestTy, + CE->getExprLoc()); + } + case CK_FloatingToIntegral: { + if (E->getType()->isVectorType() && DestTy->isVectorType()) { + // TODO: Support constrained FP intrinsics. + QualType DstElTy = DestTy->castAs<VectorType>()->getElementType(); + if (DstElTy->isSignedIntegerOrEnumerationType()) + return Builder.CreateFPToSI(Visit(E), ConvertType(DestTy), "conv"); + return Builder.CreateFPToUI(Visit(E), ConvertType(DestTy), "conv"); + } + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE); + return EmitScalarConversion(Visit(E), E->getType(), DestTy, + CE->getExprLoc()); + } + case CK_FloatingCast: { + if (E->getType()->isVectorType() && DestTy->isVectorType()) { + // TODO: Support constrained FP intrinsics. + QualType SrcElTy = E->getType()->castAs<VectorType>()->getElementType(); + QualType DstElTy = DestTy->castAs<VectorType>()->getElementType(); + if (DstElTy->castAs<BuiltinType>()->getKind() < + SrcElTy->castAs<BuiltinType>()->getKind()) + return Builder.CreateFPTrunc(Visit(E), ConvertType(DestTy), "conv"); + return Builder.CreateFPExt(Visit(E), ConvertType(DestTy), "conv"); + } + CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE); + return EmitScalarConversion(Visit(E), E->getType(), DestTy, + CE->getExprLoc()); + } case CK_FixedPointToFloating: case CK_FloatingToFixedPoint: { CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE); @@ -2466,6 +2691,17 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_IntToOCLSampler: return CGF.CGM.createOpenCLIntToSamplerConversion(E, CGF); + case CK_HLSLVectorTruncation: { + assert(DestTy->isVectorType() && "Expected dest type to be vector type"); + Value *Vec = Visit(const_cast<Expr *>(E)); + SmallVector<int, 16> Mask; + unsigned NumElts = DestTy->castAs<VectorType>()->getNumElements(); + for (unsigned I = 0; I != NumElts; ++I) + Mask.push_back(I); + + return Builder.CreateShuffleVector(Vec, Mask, "trunc"); + } + } // end of switch llvm_unreachable("unknown scalar cast"); @@ -2514,7 +2750,9 @@ llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior( StringRef Name = IsInc ? "inc" : "dec"; switch (CGF.getLangOpts().getSignedOverflowBehavior()) { case LangOptions::SOB_Defined: - return Builder.CreateAdd(InVal, Amount, Name); + if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) + return Builder.CreateAdd(InVal, Amount, Name); + [[fallthrough]]; case LangOptions::SOB_Undefined: if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) return Builder.CreateNSWAdd(InVal, Amount, Name); @@ -2555,6 +2793,8 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, llvm::PHINode *atomicPHI = nullptr; llvm::Value *value; llvm::Value *input; + llvm::Value *Previous = nullptr; + QualType SrcType = E->getType(); int amount = (isInc ? 1 : -1); bool isSubtraction = !isInc; @@ -2564,14 +2804,14 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, if (isInc && type->isBooleanType()) { llvm::Value *True = CGF.EmitToMemory(Builder.getTrue(), type); if (isPre) { - Builder.CreateStore(True, LV.getAddress(CGF), LV.isVolatileQualified()) + Builder.CreateStore(True, LV.getAddress(), LV.isVolatileQualified()) ->setAtomic(llvm::AtomicOrdering::SequentiallyConsistent); return Builder.getTrue(); } // For atomic bool increment, we just store true and return it for // preincrement, do an atomic swap with true for postincrement return Builder.CreateAtomicRMW( - llvm::AtomicRMWInst::Xchg, LV.getAddress(CGF), True, + llvm::AtomicRMWInst::Xchg, LV.getAddress(), True, llvm::AtomicOrdering::SequentiallyConsistent); } // Special case for atomic increment / decrement on integers, emit @@ -2589,10 +2829,27 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, llvm::Value *amt = CGF.EmitToMemory( llvm::ConstantInt::get(ConvertType(type), 1, true), type); llvm::Value *old = - Builder.CreateAtomicRMW(aop, LV.getAddress(CGF), amt, + Builder.CreateAtomicRMW(aop, LV.getAddress(), amt, llvm::AtomicOrdering::SequentiallyConsistent); return isPre ? Builder.CreateBinOp(op, old, amt) : old; } + // Special case for atomic increment/decrement on floats. + // Bail out non-power-of-2-sized floating point types (e.g., x86_fp80). + if (type->isFloatingType()) { + llvm::Type *Ty = ConvertType(type); + if (llvm::has_single_bit(Ty->getScalarSizeInBits())) { + llvm::AtomicRMWInst::BinOp aop = + isInc ? llvm::AtomicRMWInst::FAdd : llvm::AtomicRMWInst::FSub; + llvm::Instruction::BinaryOps op = + isInc ? llvm::Instruction::FAdd : llvm::Instruction::FSub; + llvm::Value *amt = llvm::ConstantFP::get(Ty, 1.0); + llvm::AtomicRMWInst *old = Builder.CreateAtomicRMW( + aop, LV.getAddress(), amt, + llvm::AtomicOrdering::SequentiallyConsistent); + + return isPre ? Builder.CreateBinOp(op, old, amt) : old; + } + } value = EmitLoadOfLValue(LV, E->getExprLoc()); input = value; // For every other atomic operation, we need to emit a load-op-cmpxchg loop @@ -2643,7 +2900,8 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, "base or promoted) will be signed, or the bitwidths will match."); } if (CGF.SanOpts.hasOneOf( - SanitizerKind::ImplicitIntegerArithmeticValueChange) && + SanitizerKind::ImplicitIntegerArithmeticValueChange | + SanitizerKind::ImplicitBitfieldConversion) && canPerformLossyDemotionCheck) { // While `x += 1` (for `x` with width less than int) is modeled as // promotion+arithmetics+demotion, and we can catch lossy demotion with @@ -2654,13 +2912,26 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, // the increment/decrement in the wider type, and finally // perform the demotion. This will catch lossy demotions. + // We have a special case for bitfields defined using all the bits of the + // type. In this case we need to do the same trick as for the integer + // sanitizer checks, i.e., promotion -> increment/decrement -> demotion. + value = EmitScalarConversion(value, type, promotedType, E->getExprLoc()); Value *amt = llvm::ConstantInt::get(value->getType(), amount, true); value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec"); // Do pass non-default ScalarConversionOpts so that sanitizer check is - // emitted. + // emitted if LV is not a bitfield, otherwise the bitfield sanitizer + // checks will take care of the conversion. + ScalarConversionOpts Opts; + if (!LV.isBitField()) + Opts = ScalarConversionOpts(CGF.SanOpts); + else if (CGF.SanOpts.has(SanitizerKind::ImplicitBitfieldConversion)) { + Previous = value; + SrcType = promotedType; + } + value = EmitScalarConversion(value, promotedType, type, E->getExprLoc(), - ScalarConversionOpts(CGF.SanOpts)); + Opts); // Note that signed integer inc/dec with width less than int can't // overflow because of promotion rules; we're just eliding a few steps @@ -2845,9 +3116,12 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, } // Store the updated result through the lvalue. - if (LV.isBitField()) + if (LV.isBitField()) { + Value *Src = Previous ? Previous : value; CGF.EmitStoreThroughBitfieldLValue(RValue::get(value), LV, &value); - else + CGF.EmitBitfieldConversionCheck(Src, SrcType, value, E->getType(), + LV.getBitFieldInfo(), E->getExprLoc()); + } else CGF.EmitStoreThroughLValue(RValue::get(value), LV); // If this is a postinc, return the value read from memory, otherwise use the @@ -3314,7 +3588,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( E->getExprLoc()), LHSTy); Value *OldVal = Builder.CreateAtomicRMW( - AtomicOp, LHSLV.getAddress(CGF), Amt, + AtomicOp, LHSLV.getAddress(), Amt, llvm::AtomicOrdering::SequentiallyConsistent); // Since operation is atomic, the result type is guaranteed to be the @@ -3352,8 +3626,15 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( // Convert the result back to the LHS type, // potentially with Implicit Conversion sanitizer check. - Result = EmitScalarConversion(Result, PromotionTypeCR, LHSTy, Loc, - ScalarConversionOpts(CGF.SanOpts)); + // If LHSLV is a bitfield, use default ScalarConversionOpts + // to avoid emit any implicit integer checks. + Value *Previous = nullptr; + if (LHSLV.isBitField()) { + Previous = Result; + Result = EmitScalarConversion(Result, PromotionTypeCR, LHSTy, Loc); + } else + Result = EmitScalarConversion(Result, PromotionTypeCR, LHSTy, Loc, + ScalarConversionOpts(CGF.SanOpts)); if (atomicPHI) { llvm::BasicBlock *curBlock = Builder.GetInsertBlock(); @@ -3372,9 +3653,14 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( // specially because the result is altered by the store, i.e., [C99 6.5.16p1] // 'An assignment expression has the value of the left operand after the // assignment...'. - if (LHSLV.isBitField()) + if (LHSLV.isBitField()) { + Value *Src = Previous ? Previous : Result; + QualType SrcType = E->getRHS()->getType(); + QualType DstType = E->getLHS()->getType(); CGF.EmitStoreThroughBitfieldLValue(RValue::get(Result), LHSLV, &Result); - else + CGF.EmitBitfieldConversionCheck(Src, SrcType, Result, DstType, + LHSLV.getBitFieldInfo(), E->getExprLoc()); + } else CGF.EmitStoreThroughLValue(RValue::get(Result), LHSLV); if (CGF.getLangOpts().OpenMP) @@ -3859,7 +4145,9 @@ Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &op) { if (op.Ty->isSignedIntegerOrEnumerationType()) { switch (CGF.getLangOpts().getSignedOverflowBehavior()) { case LangOptions::SOB_Defined: - return Builder.CreateAdd(op.LHS, op.RHS, "add"); + if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) + return Builder.CreateAdd(op.LHS, op.RHS, "add"); + [[fallthrough]]; case LangOptions::SOB_Undefined: if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) return Builder.CreateNSWAdd(op.LHS, op.RHS, "add"); @@ -4013,7 +4301,9 @@ Value *ScalarExprEmitter::EmitSub(const BinOpInfo &op) { if (op.Ty->isSignedIntegerOrEnumerationType()) { switch (CGF.getLangOpts().getSignedOverflowBehavior()) { case LangOptions::SOB_Defined: - return Builder.CreateSub(op.LHS, op.RHS, "sub"); + if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) + return Builder.CreateSub(op.LHS, op.RHS, "sub"); + [[fallthrough]]; case LangOptions::SOB_Undefined: if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) return Builder.CreateNSWSub(op.LHS, op.RHS, "sub"); @@ -4112,13 +4402,24 @@ Value *ScalarExprEmitter::EmitSub(const BinOpInfo &op) { return Builder.CreateExactSDiv(diffInChars, divisor, "sub.ptr.div"); } -Value *ScalarExprEmitter::GetWidthMinusOneValue(Value* LHS,Value* RHS) { +Value *ScalarExprEmitter::GetMaximumShiftAmount(Value *LHS, Value *RHS, + bool RHSIsSigned) { llvm::IntegerType *Ty; if (llvm::VectorType *VT = dyn_cast<llvm::VectorType>(LHS->getType())) Ty = cast<llvm::IntegerType>(VT->getElementType()); else Ty = cast<llvm::IntegerType>(LHS->getType()); - return llvm::ConstantInt::get(RHS->getType(), Ty->getBitWidth() - 1); + // For a given type of LHS the maximum shift amount is width(LHS)-1, however + // it can occur that width(LHS)-1 > range(RHS). Since there is no check for + // this in ConstantInt::get, this results in the value getting truncated. + // Constrain the return value to be max(RHS) in this case. + llvm::Type *RHSTy = RHS->getType(); + llvm::APInt RHSMax = + RHSIsSigned ? llvm::APInt::getSignedMaxValue(RHSTy->getScalarSizeInBits()) + : llvm::APInt::getMaxValue(RHSTy->getScalarSizeInBits()); + if (RHSMax.ult(Ty->getBitWidth())) + return llvm::ConstantInt::get(RHSTy, RHSMax); + return llvm::ConstantInt::get(RHSTy, Ty->getBitWidth() - 1); } Value *ScalarExprEmitter::ConstrainShiftValue(Value *LHS, Value *RHS, @@ -4130,7 +4431,7 @@ Value *ScalarExprEmitter::ConstrainShiftValue(Value *LHS, Value *RHS, Ty = cast<llvm::IntegerType>(LHS->getType()); if (llvm::isPowerOf2_64(Ty->getBitWidth())) - return Builder.CreateAnd(RHS, GetWidthMinusOneValue(LHS, RHS), Name); + return Builder.CreateAnd(RHS, GetMaximumShiftAmount(LHS, RHS, false), Name); return Builder.CreateURem( RHS, llvm::ConstantInt::get(RHS->getType(), Ty->getBitWidth()), Name); @@ -4157,13 +4458,15 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) { bool SanitizeBase = SanitizeSignedBase || SanitizeUnsignedBase; bool SanitizeExponent = CGF.SanOpts.has(SanitizerKind::ShiftExponent); // OpenCL 6.3j: shift values are effectively % word size of LHS. - if (CGF.getLangOpts().OpenCL) + if (CGF.getLangOpts().OpenCL || CGF.getLangOpts().HLSL) RHS = ConstrainShiftValue(Ops.LHS, RHS, "shl.mask"); else if ((SanitizeBase || SanitizeExponent) && isa<llvm::IntegerType>(Ops.LHS->getType())) { CodeGenFunction::SanitizerScope SanScope(&CGF); SmallVector<std::pair<Value *, SanitizerMask>, 2> Checks; - llvm::Value *WidthMinusOne = GetWidthMinusOneValue(Ops.LHS, Ops.RHS); + bool RHSIsSigned = Ops.rhsHasSignedIntegerRepresentation(); + llvm::Value *WidthMinusOne = + GetMaximumShiftAmount(Ops.LHS, Ops.RHS, RHSIsSigned); llvm::Value *ValidExponent = Builder.CreateICmpULE(Ops.RHS, WidthMinusOne); if (SanitizeExponent) { @@ -4181,7 +4484,7 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) { Builder.CreateCondBr(ValidExponent, CheckShiftBase, Cont); llvm::Value *PromotedWidthMinusOne = (RHS == Ops.RHS) ? WidthMinusOne - : GetWidthMinusOneValue(Ops.LHS, RHS); + : GetMaximumShiftAmount(Ops.LHS, RHS, RHSIsSigned); CGF.EmitBlock(CheckShiftBase); llvm::Value *BitsShiftedOff = Builder.CreateLShr( Ops.LHS, Builder.CreateSub(PromotedWidthMinusOne, RHS, "shl.zeros", @@ -4226,13 +4529,14 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) { RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom"); // OpenCL 6.3j: shift values are effectively % word size of LHS. - if (CGF.getLangOpts().OpenCL) + if (CGF.getLangOpts().OpenCL || CGF.getLangOpts().HLSL) RHS = ConstrainShiftValue(Ops.LHS, RHS, "shr.mask"); else if (CGF.SanOpts.has(SanitizerKind::ShiftExponent) && isa<llvm::IntegerType>(Ops.LHS->getType())) { CodeGenFunction::SanitizerScope SanScope(&CGF); - llvm::Value *Valid = - Builder.CreateICmpULE(RHS, GetWidthMinusOneValue(Ops.LHS, RHS)); + bool RHSIsSigned = Ops.rhsHasSignedIntegerRepresentation(); + llvm::Value *Valid = Builder.CreateICmpULE( + Ops.RHS, GetMaximumShiftAmount(Ops.LHS, Ops.RHS, RHSIsSigned)); EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::ShiftExponent), Ops); } @@ -4474,6 +4778,24 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E, E->getExprLoc()); } +llvm::Value *CodeGenFunction::EmitWithOriginalRHSBitfieldAssignment( + const BinaryOperator *E, Value **Previous, QualType *SrcType) { + // In case we have the integer or bitfield sanitizer checks enabled + // we want to get the expression before scalar conversion. + if (auto *ICE = dyn_cast<ImplicitCastExpr>(E->getRHS())) { + CastKind Kind = ICE->getCastKind(); + if (Kind == CK_IntegralCast || Kind == CK_LValueToRValue) { + *SrcType = ICE->getSubExpr()->getType(); + *Previous = EmitScalarExpr(ICE->getSubExpr()); + // Pass default ScalarConversionOpts to avoid emitting + // integer sanitizer checks as E refers to bitfield. + return EmitScalarConversion(*Previous, *SrcType, ICE->getType(), + ICE->getExprLoc()); + } + } + return EmitScalarExpr(E->getRHS()); +} + Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) { bool Ignore = TestAndClearIgnoreResultAssign(); @@ -4496,13 +4818,22 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) { case Qualifiers::OCL_Weak: RHS = Visit(E->getRHS()); LHS = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); - RHS = CGF.EmitARCStoreWeak(LHS.getAddress(CGF), RHS, Ignore); + RHS = CGF.EmitARCStoreWeak(LHS.getAddress(), RHS, Ignore); break; case Qualifiers::OCL_None: // __block variables need to have the rhs evaluated first, plus // this should improve codegen just a little. - RHS = Visit(E->getRHS()); + Value *Previous = nullptr; + QualType SrcType = E->getRHS()->getType(); + // Check if LHS is a bitfield, if RHS contains an implicit cast expression + // we want to extract that value and potentially (if the bitfield sanitizer + // is enabled) use it to check for an implicit conversion. + if (E->getLHS()->refersToBitField()) + RHS = CGF.EmitWithOriginalRHSBitfieldAssignment(E, &Previous, &SrcType); + else + RHS = Visit(E->getRHS()); + LHS = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); // Store the value into the LHS. Bit-fields are handled specially @@ -4511,6 +4842,12 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) { // the assignment...'. if (LHS.isBitField()) { CGF.EmitStoreThroughBitfieldLValue(RValue::get(RHS), LHS, &RHS); + // If the expression contained an implicit conversion, make sure + // to use the value before the scalar conversion. + Value *Src = Previous ? Previous : RHS; + QualType DstType = E->getLHS()->getType(); + CGF.EmitBitfieldConversionCheck(Src, SrcType, RHS, DstType, + LHS.getBitFieldInfo(), E->getExprLoc()); } else { CGF.EmitNullabilityCheck(LHS, RHS, E->getExprLoc()); CGF.EmitStoreThroughLValue(RValue::get(RHS), LHS); @@ -4855,8 +5192,13 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // If the dead side doesn't have labels we need, just emit the Live part. if (!CGF.ContainsLabel(dead)) { - if (CondExprBool) + if (CondExprBool) { + if (llvm::EnableSingleByteCoverage) { + CGF.incrementProfileCounter(lhsExpr); + CGF.incrementProfileCounter(rhsExpr); + } CGF.incrementProfileCounter(E); + } Value *Result = Visit(live); // If the live part is a throw expression, it acts like it has a void @@ -4935,7 +5277,12 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr); llvm::Value *StepV = Builder.CreateZExtOrBitCast(CondV, CGF.Int64Ty); - CGF.incrementProfileCounter(E, StepV); + if (llvm::EnableSingleByteCoverage) { + CGF.incrementProfileCounter(lhsExpr); + CGF.incrementProfileCounter(rhsExpr); + CGF.incrementProfileCounter(E); + } else + CGF.incrementProfileCounter(E, StepV); llvm::Value *LHS = Visit(lhsExpr); llvm::Value *RHS = Visit(rhsExpr); @@ -4967,7 +5314,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { if (CGF.MCDCLogOpStack.empty()) CGF.maybeUpdateMCDCTestVectorBitmap(condExpr); - CGF.incrementProfileCounter(E); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(lhsExpr); + else + CGF.incrementProfileCounter(E); + eval.begin(CGF); Value *LHS = Visit(lhsExpr); eval.end(CGF); @@ -4983,6 +5334,9 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { if (CGF.MCDCLogOpStack.empty()) CGF.maybeUpdateMCDCTestVectorBitmap(condExpr); + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(rhsExpr); + eval.begin(CGF); Value *RHS = Visit(rhsExpr); eval.end(CGF); @@ -5001,6 +5355,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { PN->addIncoming(LHS, LHSBlock); PN->addIncoming(RHS, RHSBlock); + // When single byte coverage mode is enabled, add a counter to continuation + // block. + if (llvm::EnableSingleByteCoverage) + CGF.incrementProfileCounter(E); + return PN; } @@ -5015,28 +5374,9 @@ Value *ScalarExprEmitter::VisitVAArgExpr(VAArgExpr *VE) { CGF.EmitVariablyModifiedType(Ty); Address ArgValue = Address::invalid(); - Address ArgPtr = CGF.EmitVAArg(VE, ArgValue); + RValue ArgPtr = CGF.EmitVAArg(VE, ArgValue); - llvm::Type *ArgTy = ConvertType(VE->getType()); - - // If EmitVAArg fails, emit an error. - if (!ArgPtr.isValid()) { - CGF.ErrorUnsupported(VE, "va_arg expression"); - return llvm::UndefValue::get(ArgTy); - } - - // FIXME Volatility. - llvm::Value *Val = Builder.CreateLoad(ArgPtr); - - // If EmitVAArg promoted the type, we must truncate it. - if (ArgTy != Val->getType()) { - if (ArgTy->isPointerTy() && !Val->getType()->isPointerTy()) - Val = Builder.CreateIntToPtr(Val, ArgTy); - else - Val = Builder.CreateTrunc(Val, ArgTy); - } - - return Val; + return ArgPtr.getScalarVal(); } Value *ScalarExprEmitter::VisitBlockExpr(const BlockExpr *block) { @@ -5211,7 +5551,7 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) { ConvertTypeForMem(BaseExpr->getType()->getPointeeType()); Addr = Address(EmitScalarExpr(BaseExpr), BaseTy, getPointerAlign()); } else { - Addr = EmitLValue(BaseExpr).getAddress(*this); + Addr = EmitLValue(BaseExpr).getAddress(); } // Cast the address to Class*. @@ -5492,3 +5832,16 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr, return GEPVal; } + +Address CodeGenFunction::EmitCheckedInBoundsGEP( + Address Addr, ArrayRef<Value *> IdxList, llvm::Type *elementType, + bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align, + const Twine &Name) { + if (!SanOpts.has(SanitizerKind::PointerOverflow)) + return Builder.CreateInBoundsGEP(Addr, IdxList, elementType, Align, Name); + + return RawAddress( + EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this), + IdxList, SignedIndices, IsSubtraction, Loc, Name), + elementType, Align); +} diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGGPUBuiltin.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGGPUBuiltin.cpp index e465789a003e..b2340732afeb 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGGPUBuiltin.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGGPUBuiltin.cpp @@ -136,7 +136,8 @@ RValue EmitDevicePrintfCallExpr(const CallExpr *E, CodeGenFunction *CGF, llvm::Function *Decl, bool WithSizeArg) { CodeGenModule &CGM = CGF->CGM; CGBuilderTy &Builder = CGF->Builder; - assert(E->getBuiltinCallee() == Builtin::BIprintf); + assert(E->getBuiltinCallee() == Builtin::BIprintf || + E->getBuiltinCallee() == Builtin::BI__builtin_printf); assert(E->getNumArgs() >= 1); // printf always has at least one arg. // Uses the same format as nvptx for the argument packing, but also passes @@ -178,7 +179,9 @@ RValue CodeGenFunction::EmitNVPTXDevicePrintfCallExpr(const CallExpr *E) { } RValue CodeGenFunction::EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E) { - assert(getTarget().getTriple().getArch() == llvm::Triple::amdgcn); + assert(getTarget().getTriple().isAMDGCN() || + (getTarget().getTriple().isSPIRV() && + getTarget().getTriple().getVendor() == llvm::Triple::AMD)); assert(E->getBuiltinCallee() == Builtin::BIprintf || E->getBuiltinCallee() == Builtin::BI__builtin_printf); assert(E->getNumArgs() >= 1); // printf always has at least one arg. diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp index e887d35198b3..6a6aff594fb0 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -17,7 +17,6 @@ #include "CodeGenModule.h" #include "clang/AST/Decl.h" #include "clang/Basic/TargetOptions.h" -#include "llvm/IR/IntrinsicsDirectX.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/Support/FormatVariadic.h" @@ -116,6 +115,10 @@ GlobalVariable *replaceBuffer(CGHLSLRuntime::Buffer &Buf) { } // namespace +llvm::Triple::ArchType CGHLSLRuntime::getArch() { + return CGM.getTarget().getTriple().getArch(); +} + void CGHLSLRuntime::addConstant(VarDecl *D, Buffer &CB) { if (D->getStorageClass() == SC_Static) { // For static inside cbuffer, take as global static. @@ -277,13 +280,14 @@ void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { const auto *RD = Ty->getAsCXXRecordDecl(); if (!RD) return; - const auto *Attr = RD->getAttr<HLSLResourceAttr>(); - if (!Attr) + const auto *HLSLResAttr = RD->getAttr<HLSLResourceAttr>(); + const auto *HLSLResClassAttr = RD->getAttr<HLSLResourceClassAttr>(); + if (!HLSLResAttr || !HLSLResClassAttr) return; - llvm::hlsl::ResourceClass RC = Attr->getResourceClass(); - llvm::hlsl::ResourceKind RK = Attr->getResourceKind(); - bool IsROV = Attr->getIsROV(); + llvm::hlsl::ResourceClass RC = HLSLResClassAttr->getResourceClass(); + llvm::hlsl::ResourceKind RK = HLSLResAttr->getResourceKind(); + bool IsROV = HLSLResAttr->getIsROV(); llvm::hlsl::ElementType ET = calculateElementType(CGM.getContext(), Ty); BufferResBinding Binding(D->getAttr<HLSLResourceBindingAttr>()); @@ -310,7 +314,7 @@ void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes( assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr"); const StringRef ShaderAttrKindStr = "hlsl.shader"; Fn->addFnAttr(ShaderAttrKindStr, - ShaderAttr->ConvertShaderTypeToStr(ShaderAttr->getType())); + llvm::Triple::getEnvironmentTypeName(ShaderAttr->getType())); if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) { const StringRef NumThreadsKindStr = "hlsl.numthreads"; std::string NumThreadsStr = @@ -342,8 +346,9 @@ llvm::Value *CGHLSLRuntime::emitInputSemantic(IRBuilder<> &B, return B.CreateCall(FunctionCallee(DxGroupIndex)); } if (D.hasAttr<HLSLSV_DispatchThreadIDAttr>()) { - llvm::Function *DxThreadID = CGM.getIntrinsic(Intrinsic::dx_thread_id); - return buildVectorInput(B, DxThreadID, Ty); + llvm::Function *ThreadIDIntrinsic = + CGM.getIntrinsic(getThreadIdIntrinsic()); + return buildVectorInput(B, ThreadIDIntrinsic, Ty); } assert(false && "Unhandled parameter attribute"); return nullptr; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h index bffefb66740a..4036ce711bea 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h @@ -16,7 +16,11 @@ #define LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsDirectX.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "clang/Basic/Builtins.h" #include "clang/Basic/HLSLRuntime.h" #include "llvm/ADT/SmallVector.h" @@ -26,6 +30,22 @@ #include <optional> #include <vector> +// A function generator macro for picking the right intrinsic +// for the target backend +#define GENERATE_HLSL_INTRINSIC_FUNCTION(FunctionName, IntrinsicPostfix) \ + llvm::Intrinsic::ID get##FunctionName##Intrinsic() { \ + llvm::Triple::ArchType Arch = getArch(); \ + switch (Arch) { \ + case llvm::Triple::dxil: \ + return llvm::Intrinsic::dx_##IntrinsicPostfix; \ + case llvm::Triple::spirv: \ + return llvm::Intrinsic::spv_##IntrinsicPostfix; \ + default: \ + llvm_unreachable("Intrinsic " #IntrinsicPostfix \ + " not supported by target architecture"); \ + } \ + } + namespace llvm { class GlobalVariable; class Function; @@ -48,6 +68,20 @@ class CodeGenModule; class CGHLSLRuntime { public: + //===----------------------------------------------------------------------===// + // Start of reserved area for HLSL intrinsic getters. + //===----------------------------------------------------------------------===// + + GENERATE_HLSL_INTRINSIC_FUNCTION(All, all) + GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any) + GENERATE_HLSL_INTRINSIC_FUNCTION(Lerp, lerp) + GENERATE_HLSL_INTRINSIC_FUNCTION(Rsqrt, rsqrt) + GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id) + + //===----------------------------------------------------------------------===// + // End of reserved area for HLSL intrinsic getters. + //===----------------------------------------------------------------------===// + struct BufferResBinding { // The ID like 2 in register(b2, space1). std::optional<unsigned> Reg; @@ -96,6 +130,7 @@ private: BufferResBinding &Binding); void addConstant(VarDecl *D, Buffer &CB); void addBufferDecls(const DeclContext *DC, Buffer &CB); + llvm::Triple::ArchType getArch(); llvm::SmallVector<Buffer> Buffers; }; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp index 0d4800b90a2f..6b886bd6b6d2 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp @@ -612,9 +612,9 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); const OpenCLUnrollHintAttr *OpenCLHint = dyn_cast<OpenCLUnrollHintAttr>(Attr); - + const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Attr); // Skip non loop hint attributes - if (!LH && !OpenCLHint) { + if (!LH && !OpenCLHint && !HLSLLoopHint) { continue; } @@ -635,6 +635,17 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, Option = LoopHintAttr::UnrollCount; State = LoopHintAttr::Numeric; } + } else if (HLSLLoopHint) { + ValueInt = HLSLLoopHint->getDirective(); + if (HLSLLoopHint->getSemanticSpelling() == + HLSLLoopHintAttr::Spelling::Microsoft_unroll) { + if (ValueInt == 0) + State = LoopHintAttr::Enable; + if (ValueInt > 0) { + Option = LoopHintAttr::UnrollCount; + State = LoopHintAttr::Numeric; + } + } } else if (LH) { auto *ValueExpr = LH->getValue(); if (ValueExpr) { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.h b/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.h index a1c8c7e5307f..0fe33b289130 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.h @@ -110,6 +110,10 @@ public: /// been processed. void finish(); + /// Returns the first outer loop containing this loop if any, nullptr + /// otherwise. + const LoopInfo *getParent() const { return Parent; } + private: /// Loop ID metadata. llvm::TempMDTuple TempLoopID; @@ -291,12 +295,13 @@ public: /// Set no progress for the next loop pushed. void setMustProgress(bool P) { StagedAttrs.MustProgress = P; } -private: /// Returns true if there is LoopInfo on the stack. bool hasInfo() const { return !Active.empty(); } /// Return the LoopInfo for the current loop. HasInfo should be called /// first to ensure LoopInfo is present. const LoopInfo &getInfo() const { return *Active.back(); } + +private: /// The set of attributes that will be applied to the next pushed loop. LoopAttributes StagedAttrs; /// Stack of active loops. diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGNonTrivialStruct.cpp index 75c1d7fbea84..6a02e4dbf84d 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGNonTrivialStruct.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGNonTrivialStruct.cpp @@ -366,7 +366,7 @@ template <class Derived> struct GenFuncBase { llvm::Value *SizeInBytes = CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts); llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP( - CGF.Int8Ty, DstAddr.getPointer(), SizeInBytes); + CGF.Int8Ty, DstAddr.emitRawPointer(CGF), SizeInBytes); llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock(); // Create the header block and insert the phi instructions. @@ -376,7 +376,7 @@ template <class Derived> struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur"); - PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB); + PHIs[I]->addIncoming(StartAddrs[I].emitRawPointer(CGF), PreheaderBB); } // Create the exit and loop body blocks. @@ -410,7 +410,7 @@ template <class Derived> struct GenFuncBase { // Instrs to update the destination and source addresses. // Update phi instructions. NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize); - PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB); + PHIs[I]->addIncoming(NewAddrs[I].emitRawPointer(CGF), LoopBB); } // Insert an unconditional branch to the header block. @@ -488,7 +488,7 @@ template <class Derived> struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { Alignments[I] = Addrs[I].getAlignment(); - Ptrs[I] = Addrs[I].getPointer(); + Ptrs[I] = Addrs[I].emitRawPointer(CallerCGF); } if (llvm::Function *F = @@ -711,7 +711,7 @@ struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> { LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT); llvm::Value *SrcVal = CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal(); - CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress(*CGF)), SrcLV); + CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV); CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT), /* isInitialization */ true); } @@ -774,7 +774,7 @@ struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> { LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT); llvm::Value *SrcVal = CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal(); - CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress(*CGF)), SrcLV); + CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV); LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT); llvm::Value *DstVal = CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal(); @@ -810,7 +810,7 @@ void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF, // such structure. void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) { GenDefaultInitialize Gen(getContext()); - Address DstPtr = Dst.getAddress(*this).withElementType(CGM.Int8PtrTy); + Address DstPtr = Dst.getAddress().withElementType(CGM.Int8PtrTy); Gen.setCGF(this); QualType QT = Dst.getType(); QT = Dst.isVolatile() ? QT.withVolatile() : QT; @@ -842,7 +842,7 @@ getSpecialFunction(G &&Gen, StringRef FuncName, QualType QT, bool IsVolatile, // Functions to emit calls to the special functions of a non-trivial C struct. void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) { bool IsVolatile = Dst.isVolatile(); - Address DstPtr = Dst.getAddress(*this); + Address DstPtr = Dst.getAddress(); QualType QT = Dst.getType(); GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext()); std::string FuncName = GenName.getName(QT, IsVolatile); @@ -866,7 +866,7 @@ std::string CodeGenFunction::getNonTrivialDestructorStr(QualType QT, void CodeGenFunction::callCStructDestructor(LValue Dst) { bool IsVolatile = Dst.isVolatile(); - Address DstPtr = Dst.getAddress(*this); + Address DstPtr = Dst.getAddress(); QualType QT = Dst.getType(); GenDestructorFuncName GenName("__destructor_", DstPtr.getAlignment(), getContext()); @@ -877,7 +877,7 @@ void CodeGenFunction::callCStructDestructor(LValue Dst) { void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) { bool IsVolatile = Dst.isVolatile() || Src.isVolatile(); - Address DstPtr = Dst.getAddress(*this), SrcPtr = Src.getAddress(*this); + Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress(); QualType QT = Dst.getType(); GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(), SrcPtr.getAlignment(), getContext()); @@ -891,7 +891,7 @@ void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src ) { bool IsVolatile = Dst.isVolatile() || Src.isVolatile(); - Address DstPtr = Dst.getAddress(*this), SrcPtr = Src.getAddress(*this); + Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress(); QualType QT = Dst.getType(); GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(), SrcPtr.getAlignment(), getContext()); @@ -902,7 +902,7 @@ void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) { bool IsVolatile = Dst.isVolatile() || Src.isVolatile(); - Address DstPtr = Dst.getAddress(*this), SrcPtr = Src.getAddress(*this); + Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress(); QualType QT = Dst.getType(); GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(), SrcPtr.getAlignment(), getContext()); @@ -916,7 +916,7 @@ void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src ) { bool IsVolatile = Dst.isVolatile() || Src.isVolatile(); - Address DstPtr = Dst.getAddress(*this), SrcPtr = Src.getAddress(*this); + Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress(); QualType QT = Dst.getType(); GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(), SrcPtr.getAlignment(), getContext()); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGObjC.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGObjC.cpp index 03fc0ec7ff54..80a64d8e4cdd 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGObjC.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGObjC.cpp @@ -94,8 +94,8 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { // and cast value to correct type Address Temporary = CreateMemTemp(SubExpr->getType()); EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true); - llvm::Value *BitCast = - Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT)); + llvm::Value *BitCast = Builder.CreateBitCast( + Temporary.emitRawPointer(*this), ConvertType(ArgQT)); Args.add(RValue::get(BitCast), ArgQT); // Create char array to store type encoding @@ -204,11 +204,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin(); const ParmVarDecl *argDecl = *PI++; QualType ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Objects.getPointer()), ArgQT); + Args.add(RValue::get(Objects, *this), ArgQT); if (DLE) { argDecl = *PI++; ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Keys.getPointer()), ArgQT); + Args.add(RValue::get(Keys, *this), ArgQT); } argDecl = *PI; ArgQT = argDecl->getType().getUnqualifiedType(); @@ -586,7 +586,7 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, method->getMethodFamily() == OMF_retain) { if (auto lvalueExpr = findWeakLValue(E->getInstanceReceiver())) { LValue lvalue = EmitLValue(lvalueExpr); - llvm::Value *result = EmitARCLoadWeakRetained(lvalue.getAddress(*this)); + llvm::Value *result = EmitARCLoadWeakRetained(lvalue.getAddress()); return AdjustObjCObjectType(*this, E->getType(), RValue::get(result)); } } @@ -827,7 +827,7 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, // sizeof (Type of Ivar), isAtomic, false); CallArgList args; - llvm::Value *dest = CGF.ReturnValue.getPointer(); + llvm::Value *dest = CGF.ReturnValue.emitRawPointer(CGF); args.add(RValue::get(dest), Context.VoidPtrTy); args.add(RValue::get(src), Context.VoidPtrTy); @@ -899,9 +899,13 @@ namespace { const ObjCPropertyImplDecl *propImpl); private: + LLVM_PREFERRED_TYPE(StrategyKind) unsigned Kind : 8; + LLVM_PREFERRED_TYPE(bool) unsigned IsAtomic : 1; + LLVM_PREFERRED_TYPE(bool) unsigned IsCopy : 1; + LLVM_PREFERRED_TYPE(bool) unsigned HasStrong : 1; CharUnits IvarSize; @@ -1143,8 +1147,8 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, callCStructCopyConstructor(Dst, Src); } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar, - AtomicHelperFn); + emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), + ivar, AtomicHelperFn); } return; } @@ -1159,7 +1163,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), + emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), ivar, AtomicHelperFn); } return; @@ -1185,7 +1189,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, llvm::Type *bitcastType = llvm::Type::getIntNTy(getLLVMContext(), ivarSize); // Perform an atomic load. This does not impose ordering constraints. - Address ivarAddr = LV.getAddress(*this); + Address ivarAddr = LV.getAddress(); ivarAddr = ivarAddr.withElementType(bitcastType); llvm::LoadInst *load = Builder.CreateLoad(ivarAddr, "load"); load->setAtomic(llvm::AtomicOrdering::Unordered); @@ -1283,14 +1287,14 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, case TEK_Scalar: { llvm::Value *value; if (propType->isReferenceType()) { - value = LV.getAddress(*this).getPointer(); + value = LV.getAddress().emitRawPointer(*this); } else { // We want to load and autoreleaseReturnValue ARC __weak ivars. if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { if (getLangOpts().ObjCAutoRefCount) { value = emitARCRetainLoadOfScalar(*this, LV, ivarType); } else { - value = EmitARCLoadWeak(LV.getAddress(*this)); + value = EmitARCLoadWeak(LV.getAddress()); } // Otherwise we want to do a simple load, suppressing the @@ -1473,7 +1477,7 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl, LValue ivarLValue = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, /*quals*/ 0); - Address ivarAddr = ivarLValue.getAddress(*this); + Address ivarAddr = ivarLValue.getAddress(); // Currently, all atomic accesses have to be through integer // types, so there's no point in trying to pick a prettier type. @@ -1651,7 +1655,7 @@ namespace { void Emit(CodeGenFunction &CGF, Flags flags) override { LValue lvalue = CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), addr, ivar, /*CVR*/ 0); - CGF.emitDestroy(lvalue.getAddress(CGF), ivar->getType(), destroyer, + CGF.emitDestroy(lvalue.getAddress(), ivar->getType(), destroyer, flags.isForNormalCleanup() && useEHCleanupForArray); } }; @@ -1718,7 +1722,7 @@ void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP, LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), Ivar, 0); EmitAggExpr(IvarInit->getInit(), - AggValueSlot::forLValue(LV, *this, AggValueSlot::IsDestructed, + AggValueSlot::forLValue(LV, AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap)); @@ -1785,11 +1789,10 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ static const unsigned NumItems = 16; // Fetch the countByEnumeratingWithState:objects:count: selector. - IdentifierInfo *II[] = { - &CGM.getContext().Idents.get("countByEnumeratingWithState"), - &CGM.getContext().Idents.get("objects"), - &CGM.getContext().Idents.get("count") - }; + const IdentifierInfo *II[] = { + &CGM.getContext().Idents.get("countByEnumeratingWithState"), + &CGM.getContext().Idents.get("objects"), + &CGM.getContext().Idents.get("count")}; Selector FastEnumSel = CGM.getContext().Selectors.getSelector(std::size(II), &II[0]); @@ -1817,16 +1820,14 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ CallArgList Args; // The first argument is a temporary of the enumeration-state type. - Args.add(RValue::get(StatePtr.getPointer()), - getContext().getPointerType(StateTy)); + Args.add(RValue::get(StatePtr, *this), getContext().getPointerType(StateTy)); // The second argument is a temporary array with space for NumItems // pointers. We'll actually be loading elements from the array // pointer written into the control state; this buffer is so that // collections that *aren't* backed by arrays can still queue up // batches of elements. - Args.add(RValue::get(ItemsPtr.getPointer()), - getContext().getPointerType(ItemsTy)); + Args.add(RValue::get(ItemsPtr, *this), getContext().getPointerType(ItemsTy)); // The third argument is the capacity of that temporary array. llvm::Type *NSUIntegerTy = ConvertType(getContext().getNSUIntegerType()); @@ -1951,7 +1952,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ Builder.CreateLoad(StateItemsPtr, "stateitems"); // Fetch the value at the current index from the buffer. - llvm::Value *CurrentItemPtr = Builder.CreateGEP( + llvm::Value *CurrentItemPtr = Builder.CreateInBoundsGEP( ObjCIdType, EnumStateItems, index, "currentitem.ptr"); llvm::Value *CurrentItem = Builder.CreateAlignedLoad(ObjCIdType, CurrentItemPtr, getPointerAlign()); @@ -2027,7 +2028,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // First we check in the local buffer. llvm::Value *indexPlusOne = - Builder.CreateAdd(index, llvm::ConstantInt::get(NSUIntegerTy, 1)); + Builder.CreateNUWAdd(index, llvm::ConstantInt::get(NSUIntegerTy, 1)); // If we haven't overrun the buffer yet, we can continue. // Set the branch weights based on the simplifying assumption that this is @@ -2194,7 +2195,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr, if (!fn) fn = getARCIntrinsic(IntID, CGF.CGM); - return CGF.EmitNounwindRuntimeCall(fn, addr.getPointer()); + return CGF.EmitNounwindRuntimeCall(fn, addr.emitRawPointer(CGF)); } /// Perform an operation having the following signature: @@ -2212,9 +2213,8 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, Address addr, llvm::Type *origType = value->getType(); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy) - }; + CGF.Builder.CreateBitCast(addr.emitRawPointer(CGF), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)}; llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2233,9 +2233,8 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, Address dst, Address src, fn = getARCIntrinsic(IntID, CGF.CGM); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy) - }; + CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(src.emitRawPointer(CGF), CGF.Int8PtrPtrTy)}; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2486,9 +2485,8 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr, fn = getARCIntrinsic(llvm::Intrinsic::objc_storeStrong, CGM); llvm::Value *args[] = { - Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy), - Builder.CreateBitCast(value, Int8PtrTy) - }; + Builder.CreateBitCast(addr.emitRawPointer(*this), Int8PtrPtrTy), + Builder.CreateBitCast(value, Int8PtrTy)}; EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2510,7 +2508,7 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrong(LValue dst, !isBlock && (dst.getAlignment().isZero() || dst.getAlignment() >= CharUnits::fromQuantity(PointerAlignInBytes))) { - return EmitARCStoreStrongCall(dst.getAddress(*this), newValue, ignored); + return EmitARCStoreStrongCall(dst.getAddress(), newValue, ignored); } // Otherwise, split it out. @@ -2639,7 +2637,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) { if (!fn) fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM); - EmitNounwindRuntimeCall(fn, addr.getPointer()); + EmitNounwindRuntimeCall(fn, addr.emitRawPointer(*this)); } /// void \@objc_moveWeak(i8** %dest, i8** %src) @@ -2721,7 +2719,7 @@ llvm::Value *CodeGenFunction::EmitObjCMRRAutoreleasePoolPush() { CGObjCRuntime &Runtime = CGM.getObjCRuntime(); llvm::Value *Receiver = Runtime.EmitNSAutoreleasePoolClassRef(*this); // [NSAutoreleasePool alloc] - IdentifierInfo *II = &CGM.getContext().Idents.get("alloc"); + const IdentifierInfo *II = &CGM.getContext().Idents.get("alloc"); Selector AllocSel = getContext().Selectors.getSelector(0, &II); CallArgList Args; RValue AllocRV = @@ -2768,7 +2766,7 @@ llvm::Value *CodeGenFunction::EmitObjCAllocInit(llvm::Value *value, /// Produce the code to do a primitive release. /// [tmp drain]; void CodeGenFunction::EmitObjCMRRAutoreleasePoolPop(llvm::Value *Arg) { - IdentifierInfo *II = &CGM.getContext().Idents.get("drain"); + const IdentifierInfo *II = &CGM.getContext().Idents.get("drain"); Selector DrainSel = getContext().Selectors.getSelector(0, &II); CallArgList Args; CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(), @@ -2900,7 +2898,7 @@ static TryEmitResult tryEmitARCRetainLoadOfScalar(CodeGenFunction &CGF, result = CGF.EmitLoadOfLValue(lvalue, SourceLocation()).getScalarVal(); } else { assert(type.getObjCLifetime() == Qualifiers::OCL_Weak); - result = CGF.EmitARCLoadWeakRetained(lvalue.getAddress(CGF)); + result = CGF.EmitARCLoadWeakRetained(lvalue.getAddress()); } return TryEmitResult(result, !shouldRetain); } @@ -2924,7 +2922,7 @@ static TryEmitResult tryEmitARCRetainLoadOfScalar(CodeGenFunction &CGF, SourceLocation()).getScalarVal(); // Set the source pointer to NULL. - CGF.EmitStoreOfScalar(getNullForVariable(lv.getAddress(CGF)), lv); + CGF.EmitStoreOfScalar(getNullForVariable(lv.getAddress()), lv); return TryEmitResult(result, true); } @@ -3716,8 +3714,8 @@ CodeGenFunction::GenerateObjCAtomicSetterCopyHelperFunction( if ((HelperFn = CGM.getAtomicSetterHelperFnMap(Ty))) return HelperFn; - IdentifierInfo *II - = &CGM.getContext().Idents.get("__assign_helper_atomic_property_"); + const IdentifierInfo *II = + &CGM.getContext().Idents.get("__assign_helper_atomic_property_"); QualType ReturnTy = C.VoidTy; QualType DestTy = C.getPointerType(Ty); @@ -3814,7 +3812,7 @@ llvm::Constant *CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction( if ((HelperFn = CGM.getAtomicGetterHelperFnMap(Ty))) return HelperFn; - IdentifierInfo *II = + const IdentifierInfo *II = &CGM.getContext().Idents.get("__copy_helper_atomic_property_"); QualType ReturnTy = C.VoidTy; @@ -3908,10 +3906,10 @@ llvm::Constant *CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction( llvm::Value * CodeGenFunction::EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty) { // Get selectors for retain/autorelease. - IdentifierInfo *CopyID = &getContext().Idents.get("copy"); + const IdentifierInfo *CopyID = &getContext().Idents.get("copy"); Selector CopySelector = getContext().Selectors.getNullarySelector(CopyID); - IdentifierInfo *AutoreleaseID = &getContext().Idents.get("autorelease"); + const IdentifierInfo *AutoreleaseID = &getContext().Idents.get("autorelease"); Selector AutoreleaseSelector = getContext().Selectors.getNullarySelector(AutoreleaseID); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGObjCGNU.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGObjCGNU.cpp index a36b0cdddaf0..948b10954ebb 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGObjCGNU.cpp @@ -199,8 +199,7 @@ protected: llvm::Constant *MakeConstantString(StringRef Str, const char *Name = "") { ConstantAddress Array = CGM.GetAddrOfConstantCString(std::string(Str), Name); - return llvm::ConstantExpr::getGetElementPtr(Array.getElementType(), - Array.getPointer(), Zeros); + return Array.getPointer(); } /// Emits a linkonce_odr string, whose name is the prefix followed by the @@ -221,8 +220,7 @@ protected: GV->setVisibility(llvm::GlobalValue::HiddenVisibility); ConstStr = GV; } - return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), - ConstStr, Zeros); + return ConstStr; } /// Returns a property name and encoding string. @@ -706,7 +704,8 @@ protected: llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd}; + EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -761,8 +760,8 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::FunctionCallee LookupFn = SlotLookupFn; // Store the receiver on the stack so that we can reload it later - Address ReceiverPtr = - CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); + RawAddress ReceiverPtr = + CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); Builder.CreateStore(Receiver, ReceiverPtr); llvm::Value *self; @@ -778,9 +777,9 @@ class CGObjCGNUstep : public CGObjCGNU { LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture); llvm::Value *args[] = { - EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), - EnforceType(Builder, cmd, SelectorTy), - EnforceType(Builder, self, IdTy) }; + EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), + EnforceType(Builder, cmd, SelectorTy), + EnforceType(Builder, self, IdTy)}; llvm::CallBase *slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args); slot->setOnlyReadsMemory(); slot->setMetadata(msgSendMDKind, node); @@ -800,7 +799,7 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd}; + llvm::Value *lookupArgs[] = {ObjCSuper.emitRawPointer(CGF), cmd}; llvm::CallInst *slot = CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); @@ -1221,10 +1220,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { llvm::Value *cmd, MessageSendInfo &MSI) override { // Don't access the slot unless we're trying to cache the result. CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder, - ObjCSuper.getPointer(), - PtrToObjCSuperTy), - cmd}; + llvm::Value *lookupArgs[] = { + CGObjCGNU::EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), + PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -1476,8 +1475,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { GV->setVisibility(llvm::GlobalValue::HiddenVisibility); TypesGlobal = GV; } - return llvm::ConstantExpr::getGetElementPtr(TypesGlobal->getValueType(), - TypesGlobal, Zeros); + return TypesGlobal; } llvm::Constant *GetConstantSelector(Selector Sel, const std::string &TypeEncoding) override { @@ -2071,7 +2069,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { Builder.CreateCondBr(Builder.CreateICmpEQ(selfValue, Zero), SelfIsNilBlock, ContBlock, - MDHelper.createBranchWeights(1, 1 << 20)); + MDHelper.createUnlikelyBranchWeights()); CGF.EmitBlock(SelfIsNilBlock); @@ -2106,7 +2104,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { CGF.createBasicBlock("objc_direct_method.class_initialized"); Builder.CreateCondBr(Builder.CreateICmpEQ(isInitialized, Zeros[0]), notInitializedBlock, initializedBlock, - MDHelper.createBranchWeights(1, 1 << 20)); + MDHelper.createUnlikelyBranchWeights()); CGF.EmitBlock(notInitializedBlock); Builder.SetInsertPoint(notInitializedBlock); CGF.EmitRuntimeCall(SentInitializeFn, selfValue); @@ -2186,7 +2184,8 @@ protected: llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd, + EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), + cmd, }; if (CGM.ReturnTypeUsesSRet(MSI.CallInfo)) @@ -2903,23 +2902,29 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF, break; case CodeGenOptions::Mixed: case CodeGenOptions::NonLegacy: + StringRef name = "objc_msgSend"; if (CGM.ReturnTypeUsesFPRet(ResultType)) { - imp = - CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, IdTy, true), - "objc_msgSend_fpret") - .getCallee(); + name = "objc_msgSend_fpret"; } else if (CGM.ReturnTypeUsesSRet(MSI.CallInfo)) { - // The actual types here don't matter - we're going to bitcast the - // function anyway - imp = - CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, IdTy, true), - "objc_msgSend_stret") - .getCallee(); - } else { - imp = CGM.CreateRuntimeFunction( - llvm::FunctionType::get(IdTy, IdTy, true), "objc_msgSend") - .getCallee(); + name = "objc_msgSend_stret"; + + // The address of the memory block is be passed in x8 for POD type, + // or in x0 for non-POD type (marked as inreg). + bool shouldCheckForInReg = + CGM.getContext() + .getTargetInfo() + .getTriple() + .isWindowsMSVCEnvironment() && + CGM.getContext().getTargetInfo().getTriple().isAArch64(); + if (shouldCheckForInReg && CGM.ReturnTypeHasInReg(MSI.CallInfo)) { + name = "objc_msgSend_stret2"; + } } + // The actual types here don't matter - we're going to bitcast the + // function anyway + imp = CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, IdTy, true), + name) + .getCallee(); } // Reset the receiver in case the lookup modified it @@ -4201,15 +4206,15 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF, llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, Address AddrWeakObj) { CGBuilderTy &B = CGF.Builder; - return B.CreateCall(WeakReadFn, - EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy)); + return B.CreateCall( + WeakReadFn, EnforceType(B, AddrWeakObj.emitRawPointer(CGF), PtrToIdTy)); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); B.CreateCall(WeakAssignFn, {src, dstVal}); } @@ -4218,7 +4223,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, bool threadlocal) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); // FIXME. Add threadloca assign API assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); B.CreateCall(GlobalAssignFn, {src, dstVal}); @@ -4229,7 +4234,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, llvm::Value *ivarOffset) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), IdTy); B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset}); } @@ -4237,7 +4242,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); B.CreateCall(StrongCastAssignFn, {src, dstVal}); } @@ -4246,8 +4251,8 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, Address SrcPtr, llvm::Value *Size) { CGBuilderTy &B = CGF.Builder; - llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy); - llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy); + llvm::Value *DestPtrVal = EnforceType(B, DestPtr.emitRawPointer(CGF), PtrTy); + llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.emitRawPointer(CGF), PtrTy); B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size}); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGObjCMac.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGObjCMac.cpp index 517f7cddebc1..30f3911a8b03 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGObjCMac.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGObjCMac.cpp @@ -1310,7 +1310,7 @@ private: /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - Address EmitSelectorAddr(Selector Sel); + ConstantAddress EmitSelectorAddr(Selector Sel); public: CGObjCMac(CodeGen::CodeGenModule &cgm); @@ -1538,7 +1538,7 @@ private: /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - Address EmitSelectorAddr(Selector Sel); + ConstantAddress EmitSelectorAddr(Selector Sel); /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C /// interface. The return value has type EHTypePtrTy. @@ -1555,12 +1555,12 @@ private: // Shamelessly stolen from Analysis/CFRefCount.cpp Selector GetNullarySelector(const char* name) const { - IdentifierInfo* II = &CGM.getContext().Idents.get(name); + const IdentifierInfo *II = &CGM.getContext().Idents.get(name); return CGM.getContext().Selectors.getSelector(0, &II); } Selector GetUnarySelector(const char* name) const { - IdentifierInfo* II = &CGM.getContext().Idents.get(name); + const IdentifierInfo *II = &CGM.getContext().Idents.get(name); return CGM.getContext().Selectors.getSelector(1, &II); } @@ -1593,12 +1593,20 @@ private: } bool isClassLayoutKnownStatically(const ObjCInterfaceDecl *ID) { - // NSObject is a fixed size. If we can see the @implementation of a class - // which inherits from NSObject then we know that all it's offsets also must - // be fixed. FIXME: Can we do this if see a chain of super classes with - // implementations leading to NSObject? - return ID->getImplementation() && ID->getSuperClass() && - ID->getSuperClass()->getName() == "NSObject"; + // Test a class by checking its superclasses up to + // its base class if it has one. + for (; ID; ID = ID->getSuperClass()) { + // The layout of base class NSObject + // is guaranteed to be statically known + if (ID->getIdentifier()->getName() == "NSObject") + return true; + + // If we cannot see the @implementation of a class, + // we cannot statically know the class layout. + if (!ID->getImplementation()) + return false; + } + return false; } public: @@ -2056,9 +2064,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, const ObjCMethodDecl *Method) { // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - Address ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), - "objc_super"); + RawAddress ObjCSuper = CGF.CreateTempAlloca( + ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); CGF.Builder.CreateStore(ReceiverAsObject, @@ -2493,12 +2500,12 @@ void CGObjCCommonMac::BuildRCRecordLayout(const llvm::StructLayout *RecLayout, if (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) { auto *CArray = cast<ConstantArrayType>(Array); - uint64_t ElCount = CArray->getSize().getZExtValue(); + uint64_t ElCount = CArray->getZExtSize(); assert(CArray && "only array with known element size is supported"); FQT = CArray->getElementType(); while (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) { auto *CArray = cast<ConstantArrayType>(Array); - ElCount *= CArray->getSize().getZExtValue(); + ElCount *= CArray->getZExtSize(); FQT = CArray->getElementType(); } if (FQT->isRecordType() && ElCount) { @@ -4065,7 +4072,7 @@ void CGObjCCommonMac::GenerateDirectMethodPrologue( llvm::MDBuilder MDHelper(CGM.getLLVMContext()); Builder.CreateCondBr(Builder.CreateICmpEQ(selfValue, Zero), SelfIsNilBlock, - ContBlock, MDHelper.createBranchWeights(1, 1 << 20)); + ContBlock, MDHelper.createUnlikelyBranchWeights()); CGF.EmitBlock(SelfIsNilBlock); @@ -4251,7 +4258,7 @@ namespace { CGF.EmitBlock(FinallyCallExit); CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); CGF.EmitBlock(FinallyNoCallExit); @@ -4417,7 +4424,9 @@ void FragileHazards::emitHazardsInNewBlocks() { } static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, Address V) { - if (V.isValid()) S.insert(V.getPointer()); + if (V.isValid()) + if (llvm::Value *Ptr = V.getBasePointer()) + S.insert(Ptr); } void FragileHazards::collectLocals() { @@ -4620,13 +4629,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( - ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes, + ObjCTypes.ExceptionDataTy, ExceptionData.emitRawPointer(CGF), GEPIndexes, "setjmp_buffer"); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); @@ -4665,9 +4674,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, } else { // Retrieve the exception object. We may emit multiple blocks but // nothing can cross this so the value is already in SSA form. - llvm::CallInst *Caught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer(), "caught"); + llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), + "caught"); // Push the exception to rethrow onto the EH value stack for the // benefit of any @throws in the handlers. @@ -4690,7 +4699,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Enter a new exception try block (in case a @catch block // throws an exception). CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), @@ -4821,9 +4830,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Extract the new exception and save it to the // propagating-exception slot. assert(PropagatingExnVar.isValid()); - llvm::CallInst *NewCaught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer(), "caught"); + llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), + "caught"); CGF.Builder.CreateStore(NewCaught, PropagatingExnVar); // Don't pop the catch handler; the throw already did. @@ -4853,9 +4862,8 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Otherwise, just look in the buffer for the exception to throw. } else { - llvm::CallInst *Caught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer()); + llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF)); PropagatingExn = Caught; } @@ -4898,7 +4906,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, Address AddrWeakObj) { llvm::Type* DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -4920,8 +4928,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = { src, dstVal }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -4942,8 +4950,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), @@ -4969,8 +4977,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -4989,8 +4997,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "strongassign"); @@ -4999,7 +5007,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *size) { - llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size }; + llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), + SrcPtr.emitRawPointer(CGF), size}; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -5038,12 +5047,10 @@ std::string CGObjCCommonMac::GetSectionName(StringRef Section, return ("__DATA," + Section + "," + MachOAttributes).str(); } case llvm::Triple::ELF: - assert(Section.substr(0, 2) == "__" && - "expected the name to begin with __"); + assert(Section.starts_with("__") && "expected the name to begin with __"); return Section.substr(2).str(); case llvm::Triple::COFF: - assert(Section.substr(0, 2) == "__" && - "expected the name to begin with __"); + assert(Section.starts_with("__") && "expected the name to begin with __"); return ("." + Section.substr(2) + "$B").str(); case llvm::Triple::Wasm: case llvm::Triple::GOFF: @@ -5237,7 +5244,7 @@ llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) { return CGF.Builder.CreateLoad(EmitSelectorAddr(Sel)); } -Address CGObjCMac::EmitSelectorAddr(Selector Sel) { +ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) { CharUnits Align = CGM.getPointerAlign(); llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; @@ -5248,7 +5255,7 @@ Address CGObjCMac::EmitSelectorAddr(Selector Sel) { Entry->setExternallyInitialized(true); } - return Address(Entry, ObjCTypes.SelectorPtrTy, Align); + return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); } llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { @@ -5320,7 +5327,7 @@ void IvarLayoutBuilder::visitField(const FieldDecl *field, } // Unlike incomplete arrays, constant arrays can be nested. while (auto arrayType = CGM.getContext().getAsConstantArrayType(fieldType)) { - numElts *= arrayType->getSize().getZExtValue(); + numElts *= arrayType->getZExtSize(); fieldType = arrayType->getElementType(); } @@ -6261,11 +6268,10 @@ bool CGObjCNonFragileABIMac::isVTableDispatchedSelector(Selector Sel) { VTableDispatchMethods.insert(GetUnarySelector("addObject")); // "countByEnumeratingWithState:objects:count" - IdentifierInfo *KeyIdents[] = { - &CGM.getContext().Idents.get("countByEnumeratingWithState"), - &CGM.getContext().Idents.get("objects"), - &CGM.getContext().Idents.get("count") - }; + const IdentifierInfo *KeyIdents[] = { + &CGM.getContext().Idents.get("countByEnumeratingWithState"), + &CGM.getContext().Idents.get("objects"), + &CGM.getContext().Idents.get("count")}; VTableDispatchMethods.insert( CGM.getContext().Selectors.getSelector(3, KeyIdents)); } @@ -7317,7 +7323,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, ObjCTypes.MessageRefTy, CGF.getPointerAlign()); // Update the message ref argument. - args[1].setRValue(RValue::get(mref.getPointer())); + args[1].setRValue(RValue::get(mref, CGF)); // Load the function to call from the message ref table. Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0); @@ -7546,9 +7552,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // ... // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - Address ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), - "objc_super"); + RawAddress ObjCSuper = CGF.CreateTempAlloca( + ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); @@ -7588,7 +7593,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF, return LI; } -Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { +ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; CharUnits Align = CGM.getPointerAlign(); if (!Entry) { @@ -7604,7 +7609,7 @@ Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { CGM.addCompilerUsedGlobal(Entry); } - return Address(Entry, ObjCTypes.SelectorPtrTy, Align); + return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); } /// EmitObjCIvarAssign - Code gen for assigning to a __strong object. @@ -7623,8 +7628,8 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -7644,8 +7649,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "weakassign"); @@ -7654,7 +7659,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable( CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size) { - llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size }; + llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), + SrcPtr.emitRawPointer(CGF), Size}; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -7666,7 +7672,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead( Address AddrWeakObj) { llvm::Type *DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -7688,8 +7694,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -7710,8 +7716,8 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGObjCRuntime.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGObjCRuntime.cpp index 424564f97599..01d0f35da196 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -67,7 +67,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, V, Offset, "add.ptr"); if (!Ivar->isBitField()) { - LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy); + LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V, IvarTy); return LV; } @@ -233,7 +233,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI(); if (auto *CPI = dyn_cast_or_null<llvm::CatchPadInst>(CPICandidate)) { CGF.CurrentFuncletPad = CPI; - CPI->setOperand(2, CGF.getExceptionSlot().getPointer()); + CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF)); CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI); } } @@ -405,7 +405,7 @@ bool CGObjCRuntime::canMessageReceiverBeNull(CodeGenFunction &CGF, auto self = curMethod->getSelfDecl(); if (self->getType().isConstQualified()) { if (auto LI = dyn_cast<llvm::LoadInst>(receiver->stripPointerCasts())) { - llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).getPointer(); + llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).emitRawPointer(CGF); if (selfAddr == LI->getPointerOperand()) { return false; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 4855e7410a01..a6a87ec88ee8 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "CGOpenMPRuntime.h" +#include "ABIInfoImpl.h" #include "CGCXXABI.h" #include "CGCleanup.h" #include "CGRecordLayout.h" @@ -30,6 +31,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SmallBitVector.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/Constants.h" @@ -373,7 +375,7 @@ public: /*RefersToEnclosingVariableOrCapture=*/false, VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); - PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); + PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress()); } (void)PrivScope.Privatize(); } @@ -622,7 +624,7 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF, auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, Name); - LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); + LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty); RValue InitRVal; switch (CGF.getEvaluationKind(Ty)) { case TEK_Scalar: @@ -668,8 +670,8 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, llvm::Value *SrcBegin = nullptr; if (DRD) - SrcBegin = SrcAddr.getPointer(); - llvm::Value *DestBegin = DestAddr.getPointer(); + SrcBegin = SrcAddr.emitRawPointer(CGF); + llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -742,8 +744,8 @@ LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E) { - if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E)) - return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); + if (const auto *OASE = dyn_cast<ArraySectionExpr>(E)) + return CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false); return LValue(); } @@ -800,7 +802,7 @@ void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { QualType PrivateType = getPrivateType(N); - bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref); + bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref); if (!PrivateType->isVariablyModifiedType()) { Sizes.emplace_back( CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), @@ -809,7 +811,7 @@ void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { } llvm::Value *Size; llvm::Value *SizeInChars; - auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType(); + auto *ElemType = OrigAddresses[N].first.getAddress().getElementType(); auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType); if (AsArraySection) { Size = CGF.Builder.CreatePtrDiff(ElemType, @@ -897,22 +899,22 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && !CGF.getContext().hasSameType(BaseTy, ElTy)) { if (const auto *PtrTy = BaseTy->getAs<PointerType>()) { - BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); + BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy); } else { - LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); + LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy); BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); } BaseTy = BaseTy->getPointeeType(); } return CGF.MakeAddrLValue( - BaseLV.getAddress(CGF).withElementType(CGF.ConvertTypeForMem(ElTy)), + BaseLV.getAddress().withElementType(CGF.ConvertTypeForMem(ElTy)), BaseLV.getType(), BaseLV.getBaseInfo(), CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); } static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr) { - Address Tmp = Address::invalid(); + RawAddress Tmp = RawAddress::invalid(); Address TopTmp = Address::invalid(); Address MostTopTmp = Address::invalid(); BaseTy = BaseTy.getNonReferenceType(); @@ -941,9 +943,9 @@ static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) { const VarDecl *OrigVD = nullptr; - if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) { + if (const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) { const Expr *Base = OASE->getBase()->IgnoreParenImpCasts(); - while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base)) + while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Base)) Base = TempOASE->getBase()->IgnoreParenImpCasts(); while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) Base = TempASE->getBase()->IgnoreParenImpCasts(); @@ -968,18 +970,18 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, LValue BaseLValue = loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), OriginalBaseLValue); - Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); + Address SharedAddr = SharedAddresses[N].first.getAddress(); llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( SharedAddr.getElementType(), BaseLValue.getPointer(CGF), - SharedAddr.getPointer()); + SharedAddr.emitRawPointer(CGF)); llvm::Value *PrivatePointer = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - PrivateAddr.getPointer(), SharedAddr.getType()); + PrivateAddr.emitRawPointer(CGF), SharedAddr.getType()); llvm::Value *Ptr = CGF.Builder.CreateGEP( SharedAddr.getElementType(), PrivatePointer, Adjustment); return castToBase(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), - OriginalBaseLValue.getAddress(CGF), Ptr); + OriginalBaseLValue.getAddress(), Ptr); } BaseDecls.emplace_back( cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl())); @@ -1108,11 +1110,11 @@ emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); Scope.addPrivate( In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) - .getAddress(CGF)); + .getAddress()); Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); Scope.addPrivate( Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) - .getAddress(CGF)); + .getAddress()); (void)Scope.Privatize(); if (!IsCombiner && Out->hasInit() && !CGF.isTrivialInitializer(Out->getInit())) { @@ -1557,7 +1559,7 @@ static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc( return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName); } -Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { +ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); }; auto LinkageForVariable = [&VD, this]() { @@ -1579,8 +1581,8 @@ Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { LinkageForVariable); if (!addr) - return Address::invalid(); - return Address(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); + return ConstantAddress::invalid(); + return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); } llvm::Constant * @@ -1604,7 +1606,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; return Address( @@ -1627,7 +1629,8 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit( // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. llvm::Value *Args[] = { - OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), + OMPLoc, + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( OMPBuilder.getOrCreateRuntimeFunction( @@ -1900,13 +1903,13 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, // OutlinedFn(>id, &zero_bound, CapturedStruct); Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); - Address ZeroAddrBound = + RawAddress ZeroAddrBound = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".bound.zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; // ThreadId for serialized parallels is 0. - OutlinedFnArgs.push_back(ThreadIDAddr.getPointer()); + OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF)); OutlinedFnArgs.push_back(ZeroAddrBound.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); @@ -1945,7 +1948,7 @@ Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, if (auto *OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) if (OMPRegionInfo->getThreadIDVariable()) - return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); + return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(); llvm::Value *ThreadID = getThreadID(CGF, Loc); QualType Int32Ty = @@ -2272,7 +2275,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, emitUpdateLocation(CGF, Loc), // ident_t *<loc> getThreadID(CGF, Loc), // i32 <gtid> BufSize, // size_t <buf_size> - CL.getPointer(), // void *<copyprivate list> + CL.emitRawPointer(CGF), // void *<copyprivate list> CpyFn, // void (*) (void *, void *) <copy_func> DidItVal // i32 did_it }; @@ -2552,6 +2555,15 @@ void CGOpenMPRuntime::emitForDispatchInit( Args); } +void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF, + SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; + // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(OMPBuilder.createDispatchDeinitFunction(), Args); +} + static void emitForStaticInitCall( CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, @@ -2591,10 +2603,10 @@ static void emitForStaticInitCall( ThreadId, CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, M2)), // Schedule type - Values.IL.getPointer(), // &isLastIter - Values.LB.getPointer(), // &LB - Values.UB.getPointer(), // &UB - Values.ST.getPointer(), // &Stride + Values.IL.emitRawPointer(CGF), // &isLastIter + Values.LB.emitRawPointer(CGF), // &LB + Values.UB.emitRawPointer(CGF), // &UB + Values.ST.emitRawPointer(CGF), // &Stride CGF.Builder.getIntN(Values.IVSize, 1), // Incr Chunk // Chunk }; @@ -2647,16 +2659,20 @@ void CGOpenMPRuntime::emitDistributeStaticInit( void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) { + assert((DKind == OMPD_distribute || DKind == OMPD_for || + DKind == OMPD_sections) && + "Expected distribute, for, or sections directive kind"); if (!CGF.HaveInsertPoint()) return; // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, - isOpenMPDistributeDirective(DKind) + isOpenMPDistributeDirective(DKind) || + (DKind == OMPD_target_teams_loop) ? OMP_IDENT_WORK_DISTRIBUTE - : isOpenMPLoopDirective(DKind) - ? OMP_IDENT_WORK_LOOP - : OMP_IDENT_WORK_SECTIONS), + : isOpenMPLoopDirective(DKind) + ? OMP_IDENT_WORK_LOOP + : OMP_IDENT_WORK_SECTIONS), getThreadID(CGF, Loc)}; auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); if (isOpenMPDistributeDirective(DKind) && @@ -2694,12 +2710,11 @@ llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { - emitUpdateLocation(CGF, Loc), - getThreadID(CGF, Loc), - IL.getPointer(), // &isLastIter - LB.getPointer(), // &Lower - UB.getPointer(), // &Upper - ST.getPointer() // &Stride + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + IL.emitRawPointer(CGF), // &isLastIter + LB.emitRawPointer(CGF), // &Lower + UB.emitRawPointer(CGF), // &Upper + ST.emitRawPointer(CGF) // &Stride }; llvm::Value *Call = CGF.EmitRuntimeCall( OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args); @@ -3042,9 +3057,9 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, llvm::Value *CommonArgs[] = { GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap, CGF.Builder - .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), + .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(), CGF.VoidPtrTy, CGF.Int8Ty) - .getPointer()}; + .emitRawPointer(CGF)}; SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs), std::end(CommonArgs)); if (isOpenMPTaskLoopDirective(Kind)) { @@ -3121,7 +3136,7 @@ static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, if (QualType::DestructionKind DtorKind = Field->getType().isDestructedType()) { LValue FieldLValue = CGF.EmitLValueForField(Base, Field); - CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); + CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType()); } } CGF.FinishFunction(); @@ -3229,7 +3244,7 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, LValue RefLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( - RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); + RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>()); CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); ++Counter; } @@ -3301,7 +3316,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF, } else if (ForDup) { SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); SharedRefLValue = CGF.MakeAddrLValue( - SharedRefLValue.getAddress(CGF).withAlignment( + SharedRefLValue.getAddress().withAlignment( C.getDeclAlign(OriginalVD)), SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl), SharedRefLValue.getTBAAInfo()); @@ -3325,8 +3340,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF, // Initialize firstprivate array using element-by-element // initialization. CGF.EmitOMPAggregateAssign( - PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), - Type, + PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type, [&CGF, Elem, Init, &CapturesInfo](Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the initialization. @@ -3343,7 +3357,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF, } } else { CodeGenFunction::OMPPrivateScope InitScope(CGF); - InitScope.addPrivate(Elem, SharedRefLValue.getAddress(CGF)); + InitScope.addPrivate(Elem, SharedRefLValue.getAddress()); (void)InitScope.Privatize(); CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); CGF.EmitExprAsInit(Init, VD, PrivateLValue, @@ -3504,7 +3518,7 @@ public: HelperData.CounterVD->getType()); // Counter = 0; CGF.EmitStoreOfScalar( - llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0), + llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0), CLVal); CodeGenFunction::JumpDest &ContDest = ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); @@ -3566,12 +3580,12 @@ getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); } } else if (const auto *ASE = - dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) { - LValue UpAddrLVal = - CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); - Address UpAddrAddress = UpAddrLVal.getAddress(CGF); + dyn_cast<ArraySectionExpr>(E->IgnoreParenImpCasts())) { + LValue UpAddrLVal = CGF.EmitArraySectionExpr(ASE, /*IsLowerBound=*/false); + Address UpAddrAddress = UpAddrLVal.getAddress(); llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( - UpAddrAddress.getElementType(), UpAddrAddress.getPointer(), /*Idx0=*/1); + UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF), + /*Idx0=*/1); llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); @@ -3885,8 +3899,9 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *Size; std::tie(Addr, Size) = getPointerAndSize(CGF, E); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - LValue Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); + LValue Base = + CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx), + KmpTaskAffinityInfoTy); // affs[i].base_addr = &<Affinities[i].second>; LValue BaseAddrLVal = CGF.EmitLValueForField( Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); @@ -3907,7 +3922,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); llvm::Value *GTid = getThreadID(CGF, Loc); llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - AffinitiesArray.getPointer(), CGM.VoidPtrTy); + AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy); // FIXME: Emit the function and ignore its result for now unless the // runtime function is properly implemented. (void)CGF.EmitRuntimeCall( @@ -3918,8 +3933,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( NewTask, KmpTaskTWithPrivatesPtrTy); - LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, - KmpTaskTWithPrivatesQTy); + LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy, + KmpTaskTWithPrivatesQTy); LValue TDBase = CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); // Fill the data in the resulting kmp_task_t record. @@ -4040,11 +4055,11 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl()); QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy); LValue Base = CGF.EmitLoadOfPointerLValue( - DepobjLVal.getAddress(CGF).withElementType( + DepobjLVal.getAddress().withElementType( CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), KmpDependInfoPtrTy->castAs<PointerType>()); Address DepObjAddr = CGF.Builder.CreateGEP( - Base.getAddress(CGF), + CGF, Base.getAddress(), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4094,7 +4109,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue &PosLVal = *Pos.get<LValue *>(); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); + CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy); } // deps[i].base_addr = &<Dependencies[i].second>; LValue BaseAddrLVal = CGF.EmitLValueForField( @@ -4151,7 +4166,7 @@ SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes( CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"), C.getUIntPtrType()); CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0), - NumLVal.getAddress(CGF)); + NumLVal.getAddress()); llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); CGF.EmitStoreOfScalar(Add, NumLVal); @@ -4192,8 +4207,8 @@ void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, ElSize, CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); - CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); + Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos); + CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(), Size); // Increase pos. // pos += size; @@ -4244,14 +4259,18 @@ std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause( // Include number of iterations, if any. if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) { + llvm::Value *ClauseIteratorSpace = + llvm::ConstantInt::get(CGF.IntPtrTy, 1); for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) { llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); - llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul( - Sz, llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size())); - NumOfRegularWithIterators = - CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps); + ClauseIteratorSpace = CGF.Builder.CreateNUWMul(Sz, ClauseIteratorSpace); } + llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul( + ClauseIteratorSpace, + llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size())); + NumOfRegularWithIterators = + CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps); HasRegularWithIterators = true; continue; } @@ -4420,14 +4439,14 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, ASTContext &C = CGM.getContext(); QualType FlagsTy; getDependTypes(C, KmpDependInfoTy, FlagsTy); - LValue Base = CGF.EmitLoadOfPointerLValue( - DepobjLVal.getAddress(CGF), C.VoidPtrTy.castAs<PointerType>()); + LValue Base = CGF.EmitLoadOfPointerLValue(DepobjLVal.getAddress(), + C.VoidPtrTy.castAs<PointerType>()); QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy); Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), + Base.getAddress(), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), CGF.ConvertTypeForMem(KmpDependInfoTy)); llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.getPointer(), + Addr.getElementType(), Addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, CGF.VoidPtrTy); @@ -4455,10 +4474,10 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, LValue Base; std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); - Address Begin = Base.getAddress(CGF); + Address Begin = Base.getAddress(); // Cast from pointer to array type to pointer to single element. - llvm::Value *End = CGF.Builder.CreateGEP( - Begin.getElementType(), Begin.getPointer(), NumDeps); + llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(), + Begin.emitRawPointer(CGF), NumDeps); // The basic structure here is a while-do loop. llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); @@ -4466,7 +4485,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, CGF.EmitBlock(BodyBB); llvm::PHINode *ElementPHI = CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); - ElementPHI->addIncoming(Begin.getPointer(), EntryBB); + ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB); Begin = Begin.withPointer(ElementPHI, KnownNonNull); Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4480,12 +4499,12 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, FlagsLVal); // Shift the address forward by one element. - Address ElementNext = - CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); - ElementPHI->addIncoming(ElementNext.getPointer(), - CGF.Builder.GetInsertBlock()); + llvm::Value *ElementNext = + CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext") + .emitRawPointer(CGF); + ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock()); llvm::Value *IsEmpty = - CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); + CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty"); CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Done. CGF.EmitBlock(DoneBB, /*IsFinished=*/true); @@ -4528,7 +4547,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepTaskArgs[1] = ThreadID; DepTaskArgs[2] = NewTask; DepTaskArgs[3] = NumOfElements; - DepTaskArgs[4] = DependenciesArray.getPointer(); + DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF); DepTaskArgs[5] = CGF.Builder.getInt32(0); DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } @@ -4560,7 +4579,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -4641,24 +4660,21 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound)); const auto *LBVar = cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl()); - CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), - LBLVal.getQuals(), + CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(), /*IsInitializer=*/true); LValue UBLVal = CGF.EmitLValueForField( Result.TDBase, *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound)); const auto *UBVar = cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl()); - CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), - UBLVal.getQuals(), + CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(), /*IsInitializer=*/true); LValue StLVal = CGF.EmitLValueForField( Result.TDBase, *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride)); const auto *StVar = cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl()); - CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), - StLVal.getQuals(), + CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(), /*IsInitializer=*/true); // Store reductions address. LValue RedLVal = CGF.EmitLValueForField( @@ -4667,7 +4683,7 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, if (Data.Reductions) { CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); } else { - CGF.EmitNullInitialization(RedLVal.getAddress(CGF), + CGF.EmitNullInitialization(RedLVal.getAddress(), CGF.getContext().VoidPtrTy); } enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 }; @@ -4722,8 +4738,8 @@ static void EmitOMPAggregateReduction( const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); - llvm::Value *RHSBegin = RHSAddr.getPointer(); - llvm::Value *LHSBegin = LHSAddr.getPointer(); + llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF); + llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF); // Cast from pointer to array type to pointer to single element. llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); @@ -4987,7 +5003,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, QualType ReductionArrayTy = C.getConstantArrayType( C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); - Address ReductionList = + RawAddress ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); const auto *IPriv = Privates.begin(); unsigned Idx = 0; @@ -5459,7 +5475,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( C.getConstantArrayType(RDType, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); // kmp_task_red_input_t .rd_input.[Size]; - Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); + RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs, Data.ReductionCopies, Data.ReductionOps); for (unsigned Cnt = 0; Cnt < Size; ++Cnt) { @@ -5470,7 +5486,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs, /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, ".rd_input.gep."); - LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); + LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType); // ElemLVal.reduce_shar = &Shareds[Cnt]; LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); RCG.emitSharedOrigLValue(CGF, Cnt); @@ -5517,8 +5533,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true), FlagsLVal); } else - CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), - FlagsLVal.getType()); + CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType()); } if (Data.IsReductionWithTaskMod) { // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int @@ -5626,7 +5641,7 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -5845,11 +5860,11 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, .getLimitedValue()); LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy, CGF.VoidPtrTy); + AllocatorTraitsLVal.getAddress(), CGF.VoidPtrPtrTy, CGF.VoidPtrTy); AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, AllocatorTraitsLVal.getBaseInfo(), AllocatorTraitsLVal.getTBAAInfo()); - llvm::Value *Traits = Addr.getPointer(); + llvm::Value *Traits = Addr.emitRawPointer(CGF); llvm::Value *AllocatorVal = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -6666,8 +6681,8 @@ private: // Given that an array section is considered a built-in type, we need to // do the calculation based on the length of the section instead of relying // on CGF.getTypeSize(E->getType()). - if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) { - QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType( + if (const auto *OAE = dyn_cast<ArraySectionExpr>(E)) { + QualType BaseTy = ArraySectionExpr::getBaseOriginalType( OAE->getBase()->IgnoreParenImpCasts()) .getCanonicalType(); @@ -6773,7 +6788,7 @@ private: /// Return true if the provided expression is a final array section. A /// final array section, is one whose length can't be proved to be one. bool isFinalArraySectionExpression(const Expr *E) const { - const auto *OASE = dyn_cast<OMPArraySectionExpr>(E); + const auto *OASE = dyn_cast<ArraySectionExpr>(E); // It is not an array section and therefore not a unity-size one. if (!OASE) @@ -6789,11 +6804,11 @@ private: // for this dimension. Also, we should always expect a length if the // base type is pointer. if (!Length) { - QualType BaseQTy = OMPArraySectionExpr::getBaseOriginalType( + QualType BaseQTy = ArraySectionExpr::getBaseOriginalType( OASE->getBase()->IgnoreParenImpCasts()) .getCanonicalType(); if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr())) - return ATy->getSize().getSExtValue() != 1; + return ATy->getSExtSize() != 1; // If we don't have a constant dimension length, we have to consider // the current section as having any size, so it is not necessarily // unitary. If it happen to be unity size, that's user fault. @@ -6825,7 +6840,8 @@ private: const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr, const Expr *MapExpr = nullptr, ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef> - OverlappedElements = std::nullopt) const { + OverlappedElements = std::nullopt, + bool AreBothBasePtrAndPteeMapped = false) const { // The following summarizes what has to be generated for each map and the // types below. The generated information is expressed in this order: // base pointer, section pointer, size, flags @@ -7001,6 +7017,10 @@ private: // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO // (*) the struct this entry pertains to is the 4th element in the list // of arguments, hence MEMBER_OF(4) + // + // map(p, p[:100]) + // ===> map(p[:100]) + // &p, &p[0], 100*sizeof(float), TARGET_PARAM | PTR_AND_OBJ | TO | FROM // Track if the map information being generated is the first for a capture. bool IsCaptureFirstInfo = IsFirstComponentList; @@ -7021,9 +7041,11 @@ private: Address BP = Address::invalid(); const Expr *AssocExpr = I->getAssociatedExpression(); const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr); - const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr); + const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr); const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr); + if (AreBothBasePtrAndPteeMapped && std::next(I) == CE) + return; if (isa<MemberExpr>(AssocExpr)) { // The base is the 'this' pointer. The content of the pointer is going // to be the base of the field being mapped. @@ -7031,7 +7053,7 @@ private: } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) || (OASE && isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) { - BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); + BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(); } else if (OAShE && isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) { BP = Address( @@ -7041,7 +7063,7 @@ private: } else { // The base is the reference to the variable. // BP = &Var. - BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); + BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(); if (const auto *VD = dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) { if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = @@ -7066,8 +7088,9 @@ private: // can be associated with the combined storage if shared memory mode is // active or the base declaration is not global variable. const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration()); - if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || - !VD || VD->hasLocalStorage()) + if (!AreBothBasePtrAndPteeMapped && + (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || + !VD || VD->hasLocalStorage())) BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); else FirstPointerInComplexData = true; @@ -7173,14 +7196,14 @@ private: // special treatment for array sections given that they are built-in // types. const auto *OASE = - dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression()); + dyn_cast<ArraySectionExpr>(I->getAssociatedExpression()); const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression()); const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression()); const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression()); bool IsPointer = OAShE || - (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE) + (OASE && ArraySectionExpr::getBaseOriginalType(OASE) .getCanonicalType() ->isAnyPointerType()) || I->getAssociatedExpression()->getType()->isAnyPointerType(); @@ -7201,7 +7224,7 @@ private: assert((Next == CE || isa<MemberExpr>(Next->getAssociatedExpression()) || isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || - isa<OMPArraySectionExpr>(Next->getAssociatedExpression()) || + isa<ArraySectionExpr>(Next->getAssociatedExpression()) || isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) || isa<UnaryOperator>(Next->getAssociatedExpression()) || isa<BinaryOperator>(Next->getAssociatedExpression())) && @@ -7239,13 +7262,13 @@ private: LValue BaseLVal = EmitMemberExprBase(CGF, ME); LowestElem = CGF.EmitLValueForFieldInitialization( BaseLVal, cast<FieldDecl>(MapDecl)) - .getAddress(CGF); + .getAddress(); LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) - .getAddress(CGF); + .getAddress(); } else { LowestElem = LB = CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) - .getAddress(CGF); + .getAddress(); } // If this component is a pointer inside the base struct then we don't @@ -7303,23 +7326,25 @@ private: LValue BaseLVal = EmitMemberExprBase(CGF, ME); ComponentLB = CGF.EmitLValueForFieldInitialization(BaseLVal, FD) - .getAddress(CGF); + .getAddress(); } else { ComponentLB = CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) - .getAddress(CGF); + .getAddress(); } - Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer()); + llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF); + llvm::Value *LBPtr = LB.emitRawPointer(CGF); + Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr, + LBPtr); break; } } assert(Size && "Failed to determine structure size"); CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7329,13 +7354,14 @@ private: LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); } CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); + llvm::Value *LBPtr = LB.emitRawPointer(CGF); Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), - LB.getPointer()); + CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF), + LBPtr); CombinedInfo.Sizes.push_back( CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7353,20 +7379,21 @@ private: (Next == CE && MapType != OMPC_MAP_unknown)) { if (!IsMappingWholeStruct) { CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1); } else { StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer()); + StructBaseCombinedInfo.BasePointers.push_back( + BP.emitRawPointer(CGF)); StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - StructBaseCombinedInfo.Pointers.push_back(LB.getPointer()); + StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); StructBaseCombinedInfo.NonContigInfo.Dims.push_back( @@ -7385,11 +7412,13 @@ private: // same expression except for the first one. We also need to signal // this map is the first one that relates with the current capture // (there is a set of entries for each capture). - OpenMPOffloadMappingFlags Flags = getMapTypeBits( - MapType, MapModifiers, MotionModifiers, IsImplicit, - !IsExpressionFirstInfo || RequiresReference || - FirstPointerInComplexData || IsMemberReference, - IsCaptureFirstInfo && !RequiresReference, IsNonContiguous); + OpenMPOffloadMappingFlags Flags = + getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit, + !IsExpressionFirstInfo || RequiresReference || + FirstPointerInComplexData || IsMemberReference, + AreBothBasePtrAndPteeMapped || + (IsCaptureFirstInfo && !RequiresReference), + IsNonContiguous); if (!IsExpressionFirstInfo || IsMemberReference) { // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well, @@ -7429,8 +7458,8 @@ private: PartialStruct.LowestElem = {FieldIndex, LowestElem}; if (IsFinalArraySection) { Address HB = - CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) - .getAddress(CGF); + CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false) + .getAddress(); PartialStruct.HighestElem = {FieldIndex, HB}; } else { PartialStruct.HighestElem = {FieldIndex, LowestElem}; @@ -7442,8 +7471,8 @@ private: } else if (FieldIndex > PartialStruct.HighestElem.first) { if (IsFinalArraySection) { Address HB = - CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) - .getAddress(CGF); + CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false) + .getAddress(); PartialStruct.HighestElem = {FieldIndex, HB}; } else { PartialStruct.HighestElem = {FieldIndex, LowestElem}; @@ -7500,12 +7529,12 @@ private: for (const OMPClauseMappableExprCommon::MappableComponent &Component : Components) { const Expr *AssocExpr = Component.getAssociatedExpression(); - const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr); + const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr); if (!OASE) continue; - QualType Ty = OMPArraySectionExpr::getBaseOriginalType(OASE->getBase()); + QualType Ty = ArraySectionExpr::getBaseOriginalType(OASE->getBase()); auto *CAT = Context.getAsConstantArrayType(Ty); auto *VAT = Context.getAsVariableArrayType(Ty); @@ -7543,8 +7572,8 @@ private: // it. if (DimSizes.size() < Components.size() - 1) { if (CAT) - DimSizes.push_back(llvm::ConstantInt::get( - CGF.Int64Ty, CAT->getSize().getZExtValue())); + DimSizes.push_back( + llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize())); else if (VAT) DimSizes.push_back(CGF.Builder.CreateIntCast( CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, @@ -7579,7 +7608,7 @@ private: continue; } - const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr); + const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr); if (!OASE) continue; @@ -7705,12 +7734,15 @@ private: for (const auto &I : RD->bases()) { if (I.isVirtual()) continue; - const auto *Base = I.getType()->getAsCXXRecordDecl(); + + QualType BaseTy = I.getType(); + const auto *Base = BaseTy->getAsCXXRecordDecl(); // Ignore empty bases. - if (Base->isEmpty() || CGF.getContext() - .getASTRecordLayout(Base) - .getNonVirtualSize() - .isZero()) + if (isEmptyRecordForLayout(CGF.getContext(), BaseTy) || + CGF.getContext() + .getASTRecordLayout(Base) + .getNonVirtualSize() + .isZero()) continue; unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base); @@ -7718,10 +7750,12 @@ private: } // Fill in virtual bases. for (const auto &I : RD->vbases()) { - const auto *Base = I.getType()->getAsCXXRecordDecl(); + QualType BaseTy = I.getType(); // Ignore empty bases. - if (Base->isEmpty()) + if (isEmptyRecordForLayout(CGF.getContext(), BaseTy)) continue; + + const auto *Base = BaseTy->getAsCXXRecordDecl(); unsigned FieldIndex = RL.getVirtualBaseIndex(Base); if (RecordLayout[FieldIndex]) continue; @@ -7732,7 +7766,8 @@ private: for (const auto *Field : RD->fields()) { // Fill in non-bitfields. (Bitfields always use a zero pattern, which we // will fill in later.) - if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { + if (!Field->isBitField() && + !isEmptyFieldForLayout(CGF.getContext(), Field)) { unsigned FieldIndex = RL.getLLVMFieldNo(Field); RecordLayout[FieldIndex] = Field; } @@ -8011,6 +8046,21 @@ private: MapCombinedInfoTy StructBaseCurInfo; const Decl *D = Data.first; const ValueDecl *VD = cast_or_null<ValueDecl>(D); + bool HasMapBasePtr = false; + bool HasMapArraySec = false; + if (VD && VD->getType()->isAnyPointerType()) { + for (const auto &M : Data.second) { + HasMapBasePtr = any_of(M, [](const MapInfo &L) { + return isa_and_present<DeclRefExpr>(L.VarRef); + }); + HasMapArraySec = any_of(M, [](const MapInfo &L) { + return isa_and_present<ArraySectionExpr, ArraySubscriptExpr>( + L.VarRef); + }); + if (HasMapBasePtr && HasMapArraySec) + break; + } + } for (const auto &M : Data.second) { for (const MapInfo &L : M) { assert(!L.Components.empty() && @@ -8027,7 +8077,8 @@ private: CurInfo, StructBaseCurInfo, PartialStruct, /*IsFirstComponentList=*/false, L.IsImplicit, /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD, - L.VarRef); + L.VarRef, /*OverlappedElements*/ std::nullopt, + HasMapBasePtr && HasMapArraySec); // If this entry relates to a device pointer, set the relevant // declaration and add the 'return pointer' flag. @@ -8208,11 +8259,11 @@ public: } CombinedInfo.Exprs.push_back(VD); // Base is the base of the struct - CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer()); + CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); // Pointer is the address of the lowest element - llvm::Value *LB = LBAddr.getPointer(); + llvm::Value *LB = LBAddr.emitRawPointer(CGF); const CXXMethodDecl *MD = CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr; const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr; @@ -8226,7 +8277,7 @@ public: // if the this[:1] expression had appeared in a map clause with a map-type // of tofrom. // Emit this[:1] - CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer()); + CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); QualType Ty = MD->getFunctionObjectParameterType(); llvm::Value *Size = CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, @@ -8235,7 +8286,7 @@ public: } else { CombinedInfo.Pointers.push_back(LB); // Size is (addr of {highest+1} element) - (addr of lowest element) - llvm::Value *HB = HBAddr.getPointer(); + llvm::Value *HB = HBAddr.emitRawPointer(CGF); llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( HBAddr.getElementType(), HB, /*Idx0=*/1); llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); @@ -8483,6 +8534,8 @@ public: assert(CurDir.is<const OMPExecutableDirective *>() && "Expect a executable directive"); const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>(); + bool HasMapBasePtr = false; + bool HasMapArraySec = false; for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) { const auto *EI = C->getVarRefs().begin(); for (const auto L : C->decl_component_lists(VD)) { @@ -8494,6 +8547,11 @@ public: assert(VDecl == VD && "We got information for the wrong declaration??"); assert(!Components.empty() && "Not expecting declaration with no component lists."); + if (VD && E && VD->getType()->isAnyPointerType() && isa<DeclRefExpr>(E)) + HasMapBasePtr = true; + if (VD && E && VD->getType()->isAnyPointerType() && + (isa<ArraySectionExpr>(E) || isa<ArraySubscriptExpr>(E))) + HasMapArraySec = true; DeclComponentLists.emplace_back(Components, C->getMapType(), C->getMapTypeModifiers(), C->isImplicit(), Mapper, E); @@ -8676,7 +8734,9 @@ public: MapType, MapModifiers, std::nullopt, Components, CombinedInfo, StructBaseCombinedInfo, PartialStruct, IsFirstComponentList, IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper, - /*ForDeviceAddr=*/false, VD, VarRef); + /*ForDeviceAddr=*/false, VD, VarRef, + /*OverlappedElements*/ std::nullopt, + HasMapBasePtr && HasMapArraySec); IsFirstComponentList = false; } } @@ -8744,7 +8804,7 @@ public: Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( CV, ElementType, CGF.getContext().getDeclAlign(VD), AlignmentSource::Decl)); - CombinedInfo.Pointers.push_back(PtrAddr.getPointer()); + CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF)); } else { CombinedInfo.Pointers.push_back(CV); } @@ -8770,7 +8830,7 @@ static ValueDecl *getDeclFromThisExpr(const Expr *E) { if (!E) return nullptr; - if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E->IgnoreParenCasts())) + if (const auto *OASE = dyn_cast<ArraySectionExpr>(E->IgnoreParenCasts())) if (const MemberExpr *ME = dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts())) return ME->getMemberDecl(); @@ -8876,7 +8936,8 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) { OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind(); switch (D.getDirectiveKind()) { case OMPD_target: - // For now, just treat 'target teams loop' as if it's distributed. + // For now, treat 'target' with nested 'teams loop' as if it's + // distributed (target teams distribute). if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop) return NestedDir; if (DKind == OMPD_teams) { @@ -9360,7 +9421,8 @@ llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall( SizeEmitter) { OpenMPDirectiveKind Kind = D.getDirectiveKind(); const OMPExecutableDirective *TD = &D; - // Get nested teams distribute kind directive, if any. + // Get nested teams distribute kind directive, if any. For now, treat + // 'target_teams_loop' as if it's really a target_teams_distribute. if ((!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind)) && Kind != OMPD_target_teams_loop) TD = getNestedDistributeDirective(CGM.getContext(), D); @@ -9555,10 +9617,11 @@ static void emitTargetCallKernelLaunch( bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>(); unsigned NumTargetItems = InputInfo.NumberOfTargetItems; - llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer(); - llvm::Value *PointersArray = InputInfo.PointersArray.getPointer(); - llvm::Value *SizesArray = InputInfo.SizesArray.getPointer(); - llvm::Value *MappersArray = InputInfo.MappersArray.getPointer(); + llvm::Value *BasePointersArray = + InputInfo.BasePointersArray.emitRawPointer(CGF); + llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF); + llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF); + llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF); auto &&EmitTargetCallFallbackCB = [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, @@ -10100,44 +10163,6 @@ bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) { return !AlreadyEmittedTargetDecls.insert(D).second; } -llvm::Function *CGOpenMPRuntime::emitRequiresDirectiveRegFun() { - // If we don't have entries or if we are emitting code for the device, we - // don't need to do anything. - if (CGM.getLangOpts().OMPTargetTriples.empty() || - CGM.getLangOpts().OpenMPSimd || CGM.getLangOpts().OpenMPIsTargetDevice || - (OMPBuilder.OffloadInfoManager.empty() && - !HasEmittedDeclareTargetRegion && !HasEmittedTargetRegion)) - return nullptr; - - // Create and register the function that handles the requires directives. - ASTContext &C = CGM.getContext(); - - llvm::Function *RequiresRegFn; - { - CodeGenFunction CGF(CGM); - const auto &FI = CGM.getTypes().arrangeNullaryFunction(); - llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); - std::string ReqName = getName({"omp_offloading", "requires_reg"}); - RequiresRegFn = CGM.CreateGlobalInitOrCleanUpFunction(FTy, ReqName, FI); - CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); - // TODO: check for other requires clauses. - // The requires directive takes effect only when a target region is - // present in the compilation unit. Otherwise it is ignored and not - // passed to the runtime. This avoids the runtime from throwing an error - // for mismatching requires clauses across compilation units that don't - // contain at least 1 target region. - assert((HasEmittedTargetRegion || HasEmittedDeclareTargetRegion || - !OMPBuilder.OffloadInfoManager.empty()) && - "Target or declare target region expected."); - CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( - CGM.getModule(), OMPRTL___tgt_register_requires), - llvm::ConstantInt::get( - CGM.Int64Ty, OMPBuilder.Config.getRequiresFlags())); - CGF.FinishFunction(); - } - return RequiresRegFn; -} - void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, @@ -10344,15 +10369,12 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( // Source location for the ident struct llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); - llvm::Value *OffloadingArgs[] = {RTLoc, - DeviceID, - PointerNum, - InputInfo.BasePointersArray.getPointer(), - InputInfo.PointersArray.getPointer(), - InputInfo.SizesArray.getPointer(), - MapTypesArray, - MapNamesArray, - InputInfo.MappersArray.getPointer()}; + SmallVector<llvm::Value *, 13> OffloadingArgs( + {RTLoc, DeviceID, PointerNum, + InputInfo.BasePointersArray.emitRawPointer(CGF), + InputInfo.PointersArray.emitRawPointer(CGF), + InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray, + InputInfo.MappersArray.emitRawPointer(CGF)}); // Select the right runtime function call for each standalone // directive. @@ -10441,6 +10463,12 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( llvm_unreachable("Unexpected standalone target data directive."); break; } + if (HasNowait) { + OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty)); + OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy)); + OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty)); + OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy)); + } CGF.EmitRuntimeCall( OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn), OffloadingArgs); @@ -11163,7 +11191,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, getThreadID(CGF, D.getBeginLoc()), llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()), CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), + CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF), CGM.VoidPtrTy)}; llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction( @@ -11197,7 +11225,8 @@ static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, /*Volatile=*/false, Int64Ty); } llvm::Value *Args[] = { - ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; + ULoc, ThreadID, + CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)}; llvm::FunctionCallee RTLFn; llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); OMPDoacrossKind<T> ODK; @@ -11367,7 +11396,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( CGF, SourceLocation::getFromRawEncoding(LocEncoding)); Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - Addr.getPointer(), CGF.VoidPtrTy); + Addr.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); Args[2] = AllocVal; CGF.EmitRuntimeCall(RTLFn, Args); @@ -11639,7 +11668,7 @@ Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, CGF.EmitStoreOfScalar( llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), FiredLVal); - return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(CGF); + return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(); } namespace { @@ -11725,15 +11754,17 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LLIVTy, getName({UniqueDeclName, "iv"})); cast<llvm::GlobalVariable>(LastIV)->setAlignment( IVLVal.getAlignment().getAsAlign()); - LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); + LValue LastIVLVal = + CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType()); // Last value of the lastprivate conditional. // decltype(priv_a) last_a; llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable( CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); - Last->setAlignment(LVal.getAlignment().getAsAlign()); - LValue LastLVal = CGF.MakeAddrLValue( - Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType()); + cast<llvm::GlobalVariable>(Last)->setAlignment( + LVal.getAlignment().getAsAlign()); + LValue LastLVal = + CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment()); // Global loop counter. Required to handle inner parallel-for regions. // iv @@ -11823,7 +11854,7 @@ void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const FieldDecl* FiredDecl = std::get<2>(It->getSecond()); LValue PrivLVal = CGF.EmitLValue(FoundE); Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - PrivLVal.getAddress(CGF), + PrivLVal.getAddress(), CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)), CGF.ConvertTypeForMem(StructTy)); LValue BaseLVal = @@ -11906,9 +11937,8 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( // The variable was not updated in the region - exit. if (!GV) return; - LValue LPLVal = CGF.MakeAddrLValue( - Address(GV, GV->getValueType(), PrivLVal.getAlignment()), - PrivLVal.getType().getNonReferenceType()); + LValue LPLVal = CGF.MakeRawAddrLValue( + GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment()); llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); CGF.EmitStoreOfScalar(Res, PrivLVal); } @@ -12005,6 +12035,11 @@ void CGOpenMPSIMDRuntime::emitForDispatchInit( llvm_unreachable("Not supported in SIMD-only mode"); } +void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF, + SourceLocation Loc) { + llvm_unreachable("Not supported in SIMD-only mode"); +} + void CGOpenMPSIMDRuntime::emitForStaticInit( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h index b01b39abd160..f65314d014c0 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -946,6 +946,14 @@ public: unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues); + /// This is used for non static scheduled types and when the ordered + /// clause is present on the loop construct. + /// + /// \param CGF Reference to current CodeGenFunction. + /// \param Loc Clang source location. + /// + virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc); + /// Struct with the values to be passed to the static runtime function struct StaticRTInput { /// Size of the iteration variable in bits. @@ -1068,13 +1076,12 @@ public: /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - Address VDAddr, + const VarDecl *VD, Address VDAddr, SourceLocation Loc); /// Returns the address of the variable marked as declare target with link /// clause OR as declare target with to clause and unified memory. - virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD); + virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD); /// Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created @@ -1407,10 +1414,6 @@ public: /// \param GD Global to scan. virtual bool emitTargetGlobal(GlobalDecl GD); - /// Creates and returns a registration function for when at least one - /// requires directives was used in the current module. - llvm::Function *emitRequiresDirectiveRegFun(); - /// Creates all the offload entries in the current compilation unit /// along with the associated metadata. void createOffloadEntriesAndInfoMetadata(); @@ -1834,6 +1837,14 @@ public: unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override; + /// This is used for non static scheduled types and when the ordered + /// clause is present on the loop construct. + /// + /// \param CGF Reference to current CodeGenFunction. + /// \param Loc Clang source location. + /// + void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override; + /// Call the appropriate runtime routine to initialize it before start /// of loop. /// diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 299ee1460b3d..8965a14d88a6 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -92,9 +92,9 @@ static const ValueDecl *getPrivateItem(const Expr *RefExpr) { while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) Base = TempASE->getBase()->IgnoreParenImpCasts(); RefExpr = Base; - } else if (auto *OASE = dyn_cast<OMPArraySectionExpr>(RefExpr)) { + } else if (auto *OASE = dyn_cast<ArraySectionExpr>(RefExpr)) { const Expr *Base = OASE->getBase()->IgnoreParenImpCasts(); - while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base)) + while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Base)) Base = TempOASE->getBase()->IgnoreParenImpCasts(); while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) Base = TempASE->getBase()->IgnoreParenImpCasts(); @@ -501,31 +501,6 @@ public: }; } // anonymous namespace -/// Get the id of the warp in the block. -/// We assume that the warp size is 32, which is always the case -/// on the NVPTX device, to generate more efficient code. -static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { - CGBuilderTy &Bld = CGF.Builder; - unsigned LaneIDBits = - llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); - auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); - return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); -} - -/// Get the id of the current lane in the Warp. -/// We assume that the warp size is 32, which is always the case -/// on the NVPTX device, to generate more efficient code. -static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { - CGBuilderTy &Bld = CGF.Builder; - unsigned LaneIDBits = - llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); - assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device."); - unsigned LaneIDMask = ~0u >> (32u - LaneIDBits); - auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); - return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), - "nvptx_lane_id"); -} - CGOpenMPRuntimeGPU::ExecutionMode CGOpenMPRuntimeGPU::getExecutionMode() const { return CurrentExecutionMode; @@ -646,7 +621,6 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx, case OMPD_target: case OMPD_target_teams: return hasNestedSPMDDirective(Ctx, D); - case OMPD_target_teams_loop: case OMPD_target_parallel_loop: case OMPD_target_parallel: case OMPD_target_parallel_for: @@ -658,6 +632,12 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx, return true; case OMPD_target_teams_distribute: return false; + case OMPD_target_teams_loop: + // Whether this is true or not depends on how the directive will + // eventually be emitted. + if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D)) + return TTLD->canBeParallelFor(); + return false; case OMPD_parallel: case OMPD_for: case OMPD_parallel_for: @@ -1096,14 +1076,15 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( VoidPtr, VarPtrTy, VD->getName() + "_on_stack"); - LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); - Rec.second.PrivateAddr = VarAddr.getAddress(CGF); + LValue VarAddr = + CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy); + Rec.second.PrivateAddr = VarAddr.getAddress(); Rec.second.GlobalizedVal = VoidPtr; // Assign the local allocation to the newly globalized location. if (EscapedParam) { CGF.EmitStoreOfScalar(ParValue, VarAddr); - I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress(CGF)); + I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress()); } if (auto *DI = CGF.getDebugInfo()) VoidPtr->setDebugLoc(DI->SourceLocToDebugLoc(VD->getLocation())); @@ -1117,7 +1098,7 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, LValue Base = CGF.MakeAddrLValue(AddrSizePair.first, VD->getType(), CGM.getContext().getDeclAlign(VD), AlignmentSource::Decl); - I->getSecond().MappedParams->setVarAddr(CGF, VD, Base.getAddress(CGF)); + I->getSecond().MappedParams->setVarAddr(CGF, VD, Base.getAddress()); } I->getSecond().MappedParams->apply(CGF); } @@ -1206,8 +1187,8 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, bool IsBareKernel = D.getSingleClause<OMPXBareClause>(); - Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".zero.addr"); + RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, + /*Name=*/".zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; // We don't emit any thread id function call in bare kernel, but because the @@ -1215,7 +1196,7 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, if (IsBareKernel) OutlinedFnArgs.push_back(llvm::ConstantPointerNull::get(CGM.VoidPtrTy)); else - OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); + OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF)); OutlinedFnArgs.push_back(ZeroAddr.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); @@ -1289,7 +1270,7 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, llvm::ConstantInt::get(CGF.Int32Ty, -1), FnPtr, ID, - Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), + Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF), CGF.VoidPtrPtrTy), llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())}; CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -1429,1132 +1410,6 @@ static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, TBAAAccessInfo()); } -/// This function creates calls to one of two shuffle functions to copy -/// variables between lanes in a warp. -static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, - llvm::Value *Elem, - QualType ElemType, - llvm::Value *Offset, - SourceLocation Loc) { - CodeGenModule &CGM = CGF.CGM; - CGBuilderTy &Bld = CGF.Builder; - CGOpenMPRuntimeGPU &RT = - *(static_cast<CGOpenMPRuntimeGPU *>(&CGM.getOpenMPRuntime())); - llvm::OpenMPIRBuilder &OMPBuilder = RT.getOMPBuilder(); - - CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); - assert(Size.getQuantity() <= 8 && - "Unsupported bitwidth in shuffle instruction."); - - RuntimeFunction ShuffleFn = Size.getQuantity() <= 4 - ? OMPRTL___kmpc_shuffle_int32 - : OMPRTL___kmpc_shuffle_int64; - - // Cast all types to 32- or 64-bit values before calling shuffle routines. - QualType CastTy = CGF.getContext().getIntTypeForBitwidth( - Size.getQuantity() <= 4 ? 32 : 64, /*Signed=*/1); - llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); - llvm::Value *WarpSize = - Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); - - llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( - OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), ShuffleFn), - {ElemCast, Offset, WarpSize}); - - return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); -} - -static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, - Address DestAddr, QualType ElemType, - llvm::Value *Offset, SourceLocation Loc) { - CGBuilderTy &Bld = CGF.Builder; - - CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); - // Create the loop over the big sized data. - // ptr = (void*)Elem; - // ptrEnd = (void*) Elem + 1; - // Step = 8; - // while (ptr + Step < ptrEnd) - // shuffle((int64_t)*ptr); - // Step = 4; - // while (ptr + Step < ptrEnd) - // shuffle((int32_t)*ptr); - // ... - Address ElemPtr = DestAddr; - Address Ptr = SrcAddr; - Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast( - Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty); - for (int IntSize = 8; IntSize >= 1; IntSize /= 2) { - if (Size < CharUnits::fromQuantity(IntSize)) - continue; - QualType IntType = CGF.getContext().getIntTypeForBitwidth( - CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), - /*Signed=*/1); - llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); - Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo(), - IntTy); - ElemPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - ElemPtr, IntTy->getPointerTo(), IntTy); - if (Size.getQuantity() / IntSize > 1) { - llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); - llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); - llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); - llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock(); - CGF.EmitBlock(PreCondBB); - llvm::PHINode *PhiSrc = - Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); - PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB); - llvm::PHINode *PhiDest = - Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); - PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB); - Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment()); - ElemPtr = - Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment()); - llvm::Value *PtrDiff = Bld.CreatePtrDiff( - CGF.Int8Ty, PtrEnd.getPointer(), - Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(), - CGF.VoidPtrTy)); - Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), - ThenBB, ExitBB); - CGF.EmitBlock(ThenBB); - llvm::Value *Res = createRuntimeShuffleFunction( - CGF, - CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, - LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()), - IntType, Offset, Loc); - CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, - LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()); - Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); - Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); - PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB); - PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB); - CGF.EmitBranch(PreCondBB); - CGF.EmitBlock(ExitBB); - } else { - llvm::Value *Res = createRuntimeShuffleFunction( - CGF, - CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, - LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()), - IntType, Offset, Loc); - CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, - LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()); - Ptr = Bld.CreateConstGEP(Ptr, 1); - ElemPtr = Bld.CreateConstGEP(ElemPtr, 1); - } - Size = Size % IntSize; - } -} - -namespace { -enum CopyAction : unsigned { - // RemoteLaneToThread: Copy over a Reduce list from a remote lane in - // the warp using shuffle instructions. - RemoteLaneToThread, - // ThreadCopy: Make a copy of a Reduce list on the thread's stack. - ThreadCopy, -}; -} // namespace - -struct CopyOptionsTy { - llvm::Value *RemoteLaneOffset; - llvm::Value *ScratchpadIndex; - llvm::Value *ScratchpadWidth; -}; - -/// Emit instructions to copy a Reduce list, which contains partially -/// aggregated values, in the specified direction. -static void emitReductionListCopy( - CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, - ArrayRef<const Expr *> Privates, Address SrcBase, Address DestBase, - CopyOptionsTy CopyOptions = {nullptr, nullptr, nullptr}) { - - CodeGenModule &CGM = CGF.CGM; - ASTContext &C = CGM.getContext(); - CGBuilderTy &Bld = CGF.Builder; - - llvm::Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset; - - // Iterates, element-by-element, through the source Reduce list and - // make a copy. - unsigned Idx = 0; - for (const Expr *Private : Privates) { - Address SrcElementAddr = Address::invalid(); - Address DestElementAddr = Address::invalid(); - Address DestElementPtrAddr = Address::invalid(); - // Should we shuffle in an element from a remote lane? - bool ShuffleInElement = false; - // Set to true to update the pointer in the dest Reduce list to a - // newly created element. - bool UpdateDestListPtr = false; - QualType PrivatePtrType = C.getPointerType(Private->getType()); - llvm::Type *PrivateLlvmPtrType = CGF.ConvertType(PrivatePtrType); - - switch (Action) { - case RemoteLaneToThread: { - // Step 1.1: Get the address for the src element in the Reduce list. - Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx); - SrcElementAddr = CGF.EmitLoadOfPointer( - SrcElementPtrAddr.withElementType(PrivateLlvmPtrType), - PrivatePtrType->castAs<PointerType>()); - - // Step 1.2: Create a temporary to store the element in the destination - // Reduce list. - DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx); - DestElementAddr = - CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element"); - ShuffleInElement = true; - UpdateDestListPtr = true; - break; - } - case ThreadCopy: { - // Step 1.1: Get the address for the src element in the Reduce list. - Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx); - SrcElementAddr = CGF.EmitLoadOfPointer( - SrcElementPtrAddr.withElementType(PrivateLlvmPtrType), - PrivatePtrType->castAs<PointerType>()); - - // Step 1.2: Get the address for dest element. The destination - // element has already been created on the thread's stack. - DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx); - DestElementAddr = CGF.EmitLoadOfPointer( - DestElementPtrAddr.withElementType(PrivateLlvmPtrType), - PrivatePtrType->castAs<PointerType>()); - break; - } - } - - // Regardless of src and dest of copy, we emit the load of src - // element as this is required in all directions - SrcElementAddr = SrcElementAddr.withElementType( - CGF.ConvertTypeForMem(Private->getType())); - DestElementAddr = - DestElementAddr.withElementType(SrcElementAddr.getElementType()); - - // Now that all active lanes have read the element in the - // Reduce list, shuffle over the value from the remote lane. - if (ShuffleInElement) { - shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(), - RemoteLaneOffset, Private->getExprLoc()); - } else { - switch (CGF.getEvaluationKind(Private->getType())) { - case TEK_Scalar: { - llvm::Value *Elem = CGF.EmitLoadOfScalar( - SrcElementAddr, /*Volatile=*/false, Private->getType(), - Private->getExprLoc(), LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()); - // Store the source element value to the dest element address. - CGF.EmitStoreOfScalar( - Elem, DestElementAddr, /*Volatile=*/false, Private->getType(), - LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo()); - break; - } - case TEK_Complex: { - CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex( - CGF.MakeAddrLValue(SrcElementAddr, Private->getType()), - Private->getExprLoc()); - CGF.EmitStoreOfComplex( - Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()), - /*isInit=*/false); - break; - } - case TEK_Aggregate: - CGF.EmitAggregateCopy( - CGF.MakeAddrLValue(DestElementAddr, Private->getType()), - CGF.MakeAddrLValue(SrcElementAddr, Private->getType()), - Private->getType(), AggValueSlot::DoesNotOverlap); - break; - } - } - - // Step 3.1: Modify reference in dest Reduce list as needed. - // Modifying the reference in Reduce list to point to the newly - // created element. The element is live in the current function - // scope and that of functions it invokes (i.e., reduce_function). - // RemoteReduceData[i] = (void*)&RemoteElem - if (UpdateDestListPtr) { - CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast( - DestElementAddr.getPointer(), CGF.VoidPtrTy), - DestElementPtrAddr, /*Volatile=*/false, - C.VoidPtrTy); - } - - ++Idx; - } -} - -/// This function emits a helper that gathers Reduce lists from the first -/// lane of every active warp to lanes in the first warp. -/// -/// void inter_warp_copy_func(void* reduce_data, num_warps) -/// shared smem[warp_size]; -/// For all data entries D in reduce_data: -/// sync -/// If (I am the first lane in each warp) -/// Copy my local D to smem[warp_id] -/// sync -/// if (I am the first warp) -/// Copy smem[thread_id] to my local D -static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, - ArrayRef<const Expr *> Privates, - QualType ReductionArrayTy, - SourceLocation Loc) { - ASTContext &C = CGM.getContext(); - llvm::Module &M = CGM.getModule(); - - // ReduceList: thread local Reduce list. - // At the stage of the computation when this function is called, partially - // aggregated values reside in the first lane of every active warp. - ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - // NumWarps: number of warps active in the parallel region. This could - // be smaller than 32 (max warps in a CTA) for partial block reduction. - ImplicitParamDecl NumWarpsArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.getIntTypeForBitwidth(32, /* Signed */ true), - ImplicitParamKind::Other); - FunctionArgList Args; - Args.push_back(&ReduceListArg); - Args.push_back(&NumWarpsArg); - - const CGFunctionInfo &CGFI = - CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); - auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI), - llvm::GlobalValue::InternalLinkage, - "_omp_reduction_inter_warp_copy_func", &M); - CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); - Fn->setDoesNotRecurse(); - CodeGenFunction CGF(CGM); - CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); - - CGBuilderTy &Bld = CGF.Builder; - - // This array is used as a medium to transfer, one reduce element at a time, - // the data from the first lane of every warp to lanes in the first warp - // in order to perform the final step of a reduction in a parallel region - // (reduction across warps). The array is placed in NVPTX __shared__ memory - // for reduced latency, as well as to have a distinct copy for concurrently - // executing target regions. The array is declared with common linkage so - // as to be shared across compilation units. - StringRef TransferMediumName = - "__openmp_nvptx_data_transfer_temporary_storage"; - llvm::GlobalVariable *TransferMedium = - M.getGlobalVariable(TransferMediumName); - unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; - if (!TransferMedium) { - auto *Ty = llvm::ArrayType::get(CGM.Int32Ty, WarpSize); - unsigned SharedAddressSpace = C.getTargetAddressSpace(LangAS::cuda_shared); - TransferMedium = new llvm::GlobalVariable( - M, Ty, /*isConstant=*/false, llvm::GlobalVariable::WeakAnyLinkage, - llvm::UndefValue::get(Ty), TransferMediumName, - /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, - SharedAddressSpace); - CGM.addCompilerUsedGlobal(TransferMedium); - } - - auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); - // Get the CUDA thread id of the current OpenMP thread on the GPU. - llvm::Value *ThreadID = RT.getGPUThreadID(CGF); - // nvptx_lane_id = nvptx_id % warpsize - llvm::Value *LaneID = getNVPTXLaneID(CGF); - // nvptx_warp_id = nvptx_id / warpsize - llvm::Value *WarpID = getNVPTXWarpID(CGF); - - Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); - Address LocalReduceList( - Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar( - AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc, - LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo()), - ElemTy->getPointerTo()), - ElemTy, CGF.getPointerAlign()); - - unsigned Idx = 0; - for (const Expr *Private : Privates) { - // - // Warp master copies reduce element to transfer medium in __shared__ - // memory. - // - unsigned RealTySize = - C.getTypeSizeInChars(Private->getType()) - .alignTo(C.getTypeAlignInChars(Private->getType())) - .getQuantity(); - for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /=2) { - unsigned NumIters = RealTySize / TySize; - if (NumIters == 0) - continue; - QualType CType = C.getIntTypeForBitwidth( - C.toBits(CharUnits::fromQuantity(TySize)), /*Signed=*/1); - llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); - CharUnits Align = CharUnits::fromQuantity(TySize); - llvm::Value *Cnt = nullptr; - Address CntAddr = Address::invalid(); - llvm::BasicBlock *PrecondBB = nullptr; - llvm::BasicBlock *ExitBB = nullptr; - if (NumIters > 1) { - CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); - CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, - /*Volatile=*/false, C.IntTy); - PrecondBB = CGF.createBasicBlock("precond"); - ExitBB = CGF.createBasicBlock("exit"); - llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); - // There is no need to emit line number for unconditional branch. - (void)ApplyDebugLocation::CreateEmpty(CGF); - CGF.EmitBlock(PrecondBB); - Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); - llvm::Value *Cmp = - Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters)); - Bld.CreateCondBr(Cmp, BodyBB, ExitBB); - CGF.EmitBlock(BodyBB); - } - // kmpc_barrier. - CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, - /*EmitChecks=*/false, - /*ForceSimpleCall=*/true); - llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); - llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); - llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); - - // if (lane_id == 0) - llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master"); - Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB); - CGF.EmitBlock(ThenBB); - - // Reduce element = LocalReduceList[i] - Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); - llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( - ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation()); - // elemptr = ((CopyType*)(elemptrptr)) + I - Address ElemPtr(ElemPtrPtr, CopyType, Align); - if (NumIters > 1) - ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); - - // Get pointer to location in transfer medium. - // MediumPtr = &medium[warp_id] - llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP( - TransferMedium->getValueType(), TransferMedium, - {llvm::Constant::getNullValue(CGM.Int64Ty), WarpID}); - // Casting to actual data type. - // MediumPtr = (CopyType*)MediumPtrAddr; - Address MediumPtr(MediumPtrVal, CopyType, Align); - - // elem = *elemptr - //*MediumPtr = elem - llvm::Value *Elem = CGF.EmitLoadOfScalar( - ElemPtr, /*Volatile=*/false, CType, Loc, - LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo()); - // Store the source element value to the dest element address. - CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType, - LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()); - - Bld.CreateBr(MergeBB); - - CGF.EmitBlock(ElseBB); - Bld.CreateBr(MergeBB); - - CGF.EmitBlock(MergeBB); - - // kmpc_barrier. - CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, - /*EmitChecks=*/false, - /*ForceSimpleCall=*/true); - - // - // Warp 0 copies reduce element from transfer medium. - // - llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); - llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); - llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); - - Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); - llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( - AddrNumWarpsArg, /*Volatile=*/false, C.IntTy, Loc); - - // Up to 32 threads in warp 0 are active. - llvm::Value *IsActiveThread = - Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread"); - Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB); - - CGF.EmitBlock(W0ThenBB); - - // SrcMediumPtr = &medium[tid] - llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP( - TransferMedium->getValueType(), TransferMedium, - {llvm::Constant::getNullValue(CGM.Int64Ty), ThreadID}); - // SrcMediumVal = *SrcMediumPtr; - Address SrcMediumPtr(SrcMediumPtrVal, CopyType, Align); - - // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I - Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); - llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( - TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc); - Address TargetElemPtr(TargetElemPtrVal, CopyType, Align); - if (NumIters > 1) - TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); - - // *TargetElemPtr = SrcMediumVal; - llvm::Value *SrcMediumValue = - CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); - CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, - CType); - Bld.CreateBr(W0MergeBB); - - CGF.EmitBlock(W0ElseBB); - Bld.CreateBr(W0MergeBB); - - CGF.EmitBlock(W0MergeBB); - - if (NumIters > 1) { - Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1)); - CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); - CGF.EmitBranch(PrecondBB); - (void)ApplyDebugLocation::CreateEmpty(CGF); - CGF.EmitBlock(ExitBB); - } - RealTySize %= TySize; - } - ++Idx; - } - - CGF.FinishFunction(); - return Fn; -} - -/// Emit a helper that reduces data across two OpenMP threads (lanes) -/// in the same warp. It uses shuffle instructions to copy over data from -/// a remote lane's stack. The reduction algorithm performed is specified -/// by the fourth parameter. -/// -/// Algorithm Versions. -/// Full Warp Reduce (argument value 0): -/// This algorithm assumes that all 32 lanes are active and gathers -/// data from these 32 lanes, producing a single resultant value. -/// Contiguous Partial Warp Reduce (argument value 1): -/// This algorithm assumes that only a *contiguous* subset of lanes -/// are active. This happens for the last warp in a parallel region -/// when the user specified num_threads is not an integer multiple of -/// 32. This contiguous subset always starts with the zeroth lane. -/// Partial Warp Reduce (argument value 2): -/// This algorithm gathers data from any number of lanes at any position. -/// All reduced values are stored in the lowest possible lane. The set -/// of problems every algorithm addresses is a super set of those -/// addressable by algorithms with a lower version number. Overhead -/// increases as algorithm version increases. -/// -/// Terminology -/// Reduce element: -/// Reduce element refers to the individual data field with primitive -/// data types to be combined and reduced across threads. -/// Reduce list: -/// Reduce list refers to a collection of local, thread-private -/// reduce elements. -/// Remote Reduce list: -/// Remote Reduce list refers to a collection of remote (relative to -/// the current thread) reduce elements. -/// -/// We distinguish between three states of threads that are important to -/// the implementation of this function. -/// Alive threads: -/// Threads in a warp executing the SIMT instruction, as distinguished from -/// threads that are inactive due to divergent control flow. -/// Active threads: -/// The minimal set of threads that has to be alive upon entry to this -/// function. The computation is correct iff active threads are alive. -/// Some threads are alive but they are not active because they do not -/// contribute to the computation in any useful manner. Turning them off -/// may introduce control flow overheads without any tangible benefits. -/// Effective threads: -/// In order to comply with the argument requirements of the shuffle -/// function, we must keep all lanes holding data alive. But at most -/// half of them perform value aggregation; we refer to this half of -/// threads as effective. The other half is simply handing off their -/// data. -/// -/// Procedure -/// Value shuffle: -/// In this step active threads transfer data from higher lane positions -/// in the warp to lower lane positions, creating Remote Reduce list. -/// Value aggregation: -/// In this step, effective threads combine their thread local Reduce list -/// with Remote Reduce list and store the result in the thread local -/// Reduce list. -/// Value copy: -/// In this step, we deal with the assumption made by algorithm 2 -/// (i.e. contiguity assumption). When we have an odd number of lanes -/// active, say 2k+1, only k threads will be effective and therefore k -/// new values will be produced. However, the Reduce list owned by the -/// (2k+1)th thread is ignored in the value aggregation. Therefore -/// we copy the Reduce list from the (2k+1)th lane to (k+1)th lane so -/// that the contiguity assumption still holds. -static llvm::Function *emitShuffleAndReduceFunction( - CodeGenModule &CGM, ArrayRef<const Expr *> Privates, - QualType ReductionArrayTy, llvm::Function *ReduceFn, SourceLocation Loc) { - ASTContext &C = CGM.getContext(); - - // Thread local Reduce list used to host the values of data to be reduced. - ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - // Current lane id; could be logical. - ImplicitParamDecl LaneIDArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.ShortTy, - ImplicitParamKind::Other); - // Offset of the remote source lane relative to the current lane. - ImplicitParamDecl RemoteLaneOffsetArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.ShortTy, ImplicitParamKind::Other); - // Algorithm version. This is expected to be known at compile time. - ImplicitParamDecl AlgoVerArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.ShortTy, ImplicitParamKind::Other); - FunctionArgList Args; - Args.push_back(&ReduceListArg); - Args.push_back(&LaneIDArg); - Args.push_back(&RemoteLaneOffsetArg); - Args.push_back(&AlgoVerArg); - - const CGFunctionInfo &CGFI = - CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); - auto *Fn = llvm::Function::Create( - CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, - "_omp_reduction_shuffle_and_reduce_func", &CGM.getModule()); - CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); - Fn->setDoesNotRecurse(); - - CodeGenFunction CGF(CGM); - CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); - - CGBuilderTy &Bld = CGF.Builder; - - Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); - Address LocalReduceList( - Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, - C.VoidPtrTy, SourceLocation()), - ElemTy->getPointerTo()), - ElemTy, CGF.getPointerAlign()); - - Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); - llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( - AddrLaneIDArg, /*Volatile=*/false, C.ShortTy, SourceLocation()); - - Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); - llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( - AddrRemoteLaneOffsetArg, /*Volatile=*/false, C.ShortTy, SourceLocation()); - - Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); - llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( - AddrAlgoVerArg, /*Volatile=*/false, C.ShortTy, SourceLocation()); - - // Create a local thread-private variable to host the Reduce list - // from a remote lane. - Address RemoteReduceList = - CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); - - // This loop iterates through the list of reduce elements and copies, - // element by element, from a remote lane in the warp to RemoteReduceList, - // hosted on the thread's stack. - emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, - LocalReduceList, RemoteReduceList, - {/*RemoteLaneOffset=*/RemoteLaneOffsetArgVal, - /*ScratchpadIndex=*/nullptr, - /*ScratchpadWidth=*/nullptr}); - - // The actions to be performed on the Remote Reduce list is dependent - // on the algorithm version. - // - // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 && - // LaneId % 2 == 0 && Offset > 0): - // do the reduction value aggregation - // - // The thread local variable Reduce list is mutated in place to host the - // reduced data, which is the aggregated value produced from local and - // remote lanes. - // - // Note that AlgoVer is expected to be a constant integer known at compile - // time. - // When AlgoVer==0, the first conjunction evaluates to true, making - // the entire predicate true during compile time. - // When AlgoVer==1, the second conjunction has only the second part to be - // evaluated during runtime. Other conjunctions evaluates to false - // during compile time. - // When AlgoVer==2, the third conjunction has only the second part to be - // evaluated during runtime. Other conjunctions evaluates to false - // during compile time. - llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal); - - llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); - llvm::Value *CondAlgo1 = Bld.CreateAnd( - Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal)); - - llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2)); - llvm::Value *CondAlgo2 = Bld.CreateAnd( - Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1)))); - CondAlgo2 = Bld.CreateAnd( - CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0))); - - llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1); - CondReduce = Bld.CreateOr(CondReduce, CondAlgo2); - - llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); - llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); - llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); - Bld.CreateCondBr(CondReduce, ThenBB, ElseBB); - - CGF.EmitBlock(ThenBB); - // reduce_function(LocalReduceList, RemoteReduceList) - llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - LocalReduceList.getPointer(), CGF.VoidPtrTy); - llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - RemoteReduceList.getPointer(), CGF.VoidPtrTy); - CGM.getOpenMPRuntime().emitOutlinedFunctionCall( - CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); - Bld.CreateBr(MergeBB); - - CGF.EmitBlock(ElseBB); - Bld.CreateBr(MergeBB); - - CGF.EmitBlock(MergeBB); - - // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local - // Reduce list. - Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); - llvm::Value *CondCopy = Bld.CreateAnd( - Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal)); - - llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); - llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); - llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); - Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB); - - CGF.EmitBlock(CpyThenBB); - emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, - RemoteReduceList, LocalReduceList); - Bld.CreateBr(CpyMergeBB); - - CGF.EmitBlock(CpyElseBB); - Bld.CreateBr(CpyMergeBB); - - CGF.EmitBlock(CpyMergeBB); - - CGF.FinishFunction(); - return Fn; -} - -/// This function emits a helper that copies all the reduction variables from -/// the team into the provided global buffer for the reduction variables. -/// -/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data) -/// For all data entries D in reduce_data: -/// Copy local D to buffer.D[Idx] -static llvm::Value *emitListToGlobalCopyFunction( - CodeGenModule &CGM, ArrayRef<const Expr *> Privates, - QualType ReductionArrayTy, SourceLocation Loc, - const RecordDecl *TeamReductionRec, - const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> - &VarFieldMap) { - ASTContext &C = CGM.getContext(); - - // Buffer: global reduction buffer. - ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - // Idx: index of the buffer. - ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy, - ImplicitParamKind::Other); - // ReduceList: thread local Reduce list. - ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - FunctionArgList Args; - Args.push_back(&BufferArg); - Args.push_back(&IdxArg); - Args.push_back(&ReduceListArg); - - const CGFunctionInfo &CGFI = - CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); - auto *Fn = llvm::Function::Create( - CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, - "_omp_reduction_list_to_global_copy_func", &CGM.getModule()); - CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); - Fn->setDoesNotRecurse(); - CodeGenFunction CGF(CGM); - CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); - - CGBuilderTy &Bld = CGF.Builder; - - Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); - Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); - Address LocalReduceList( - Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, - C.VoidPtrTy, Loc), - ElemTy->getPointerTo()), - ElemTy, CGF.getPointerAlign()); - QualType StaticTy = C.getRecordType(TeamReductionRec); - llvm::Type *LLVMReductionsBufferTy = - CGM.getTypes().ConvertTypeForMem(StaticTy); - llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), - LLVMReductionsBufferTy->getPointerTo()); - llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), - /*Volatile=*/false, C.IntTy, - Loc)}; - unsigned Idx = 0; - for (const Expr *Private : Privates) { - // Reduce element = LocalReduceList[i] - Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); - llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( - ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation()); - // elemptr = ((CopyType*)(elemptrptr)) + I - ElemTy = CGF.ConvertTypeForMem(Private->getType()); - ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - ElemPtrPtr, ElemTy->getPointerTo()); - Address ElemPtr = - Address(ElemPtrPtr, ElemTy, C.getTypeAlignInChars(Private->getType())); - const ValueDecl *VD = cast<DeclRefExpr>(Private)->getDecl(); - // Global = Buffer.VD[Idx]; - const FieldDecl *FD = VarFieldMap.lookup(VD); - llvm::Value *BufferPtr = - Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); - LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); - Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.getPointer(), - CGF.ConvertTypeForMem(Private->getType()), - GlobAddr.getAlignment())); - switch (CGF.getEvaluationKind(Private->getType())) { - case TEK_Scalar: { - llvm::Value *V = CGF.EmitLoadOfScalar( - ElemPtr, /*Volatile=*/false, Private->getType(), Loc, - LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo()); - CGF.EmitStoreOfScalar(V, GlobLVal); - break; - } - case TEK_Complex: { - CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex( - CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc); - CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false); - break; - } - case TEK_Aggregate: - CGF.EmitAggregateCopy(GlobLVal, - CGF.MakeAddrLValue(ElemPtr, Private->getType()), - Private->getType(), AggValueSlot::DoesNotOverlap); - break; - } - ++Idx; - } - - CGF.FinishFunction(); - return Fn; -} - -/// This function emits a helper that reduces all the reduction variables from -/// the team into the provided global buffer for the reduction variables. -/// -/// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data) -/// void *GlobPtrs[]; -/// GlobPtrs[0] = (void*)&buffer.D0[Idx]; -/// ... -/// GlobPtrs[N] = (void*)&buffer.DN[Idx]; -/// reduce_function(GlobPtrs, reduce_data); -static llvm::Value *emitListToGlobalReduceFunction( - CodeGenModule &CGM, ArrayRef<const Expr *> Privates, - QualType ReductionArrayTy, SourceLocation Loc, - const RecordDecl *TeamReductionRec, - const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> - &VarFieldMap, - llvm::Function *ReduceFn) { - ASTContext &C = CGM.getContext(); - - // Buffer: global reduction buffer. - ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - // Idx: index of the buffer. - ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy, - ImplicitParamKind::Other); - // ReduceList: thread local Reduce list. - ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - FunctionArgList Args; - Args.push_back(&BufferArg); - Args.push_back(&IdxArg); - Args.push_back(&ReduceListArg); - - const CGFunctionInfo &CGFI = - CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); - auto *Fn = llvm::Function::Create( - CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, - "_omp_reduction_list_to_global_reduce_func", &CGM.getModule()); - CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); - Fn->setDoesNotRecurse(); - CodeGenFunction CGF(CGM); - CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); - - CGBuilderTy &Bld = CGF.Builder; - - Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); - QualType StaticTy = C.getRecordType(TeamReductionRec); - llvm::Type *LLVMReductionsBufferTy = - CGM.getTypes().ConvertTypeForMem(StaticTy); - llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), - LLVMReductionsBufferTy->getPointerTo()); - - // 1. Build a list of reduction variables. - // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; - Address ReductionList = - CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); - auto IPriv = Privates.begin(); - llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), - /*Volatile=*/false, C.IntTy, - Loc)}; - unsigned Idx = 0; - for (unsigned I = 0, E = Privates.size(); I < E; ++I, ++IPriv, ++Idx) { - Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); - // Global = Buffer.VD[Idx]; - const ValueDecl *VD = cast<DeclRefExpr>(*IPriv)->getDecl(); - const FieldDecl *FD = VarFieldMap.lookup(VD); - llvm::Value *BufferPtr = - Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); - LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); - Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, - C.VoidPtrTy); - if ((*IPriv)->getType()->isVariablyModifiedType()) { - // Store array size. - ++Idx; - Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); - llvm::Value *Size = CGF.Builder.CreateIntCast( - CGF.getVLASize( - CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) - .NumElts, - CGF.SizeTy, /*isSigned=*/false); - CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), - Elem); - } - } - - // Call reduce_function(GlobalReduceList, ReduceList) - llvm::Value *GlobalReduceList = ReductionList.getPointer(); - Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); - llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( - AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc); - CGM.getOpenMPRuntime().emitOutlinedFunctionCall( - CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr}); - CGF.FinishFunction(); - return Fn; -} - -/// This function emits a helper that copies all the reduction variables from -/// the team into the provided global buffer for the reduction variables. -/// -/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data) -/// For all data entries D in reduce_data: -/// Copy buffer.D[Idx] to local D; -static llvm::Value *emitGlobalToListCopyFunction( - CodeGenModule &CGM, ArrayRef<const Expr *> Privates, - QualType ReductionArrayTy, SourceLocation Loc, - const RecordDecl *TeamReductionRec, - const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> - &VarFieldMap) { - ASTContext &C = CGM.getContext(); - - // Buffer: global reduction buffer. - ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - // Idx: index of the buffer. - ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy, - ImplicitParamKind::Other); - // ReduceList: thread local Reduce list. - ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - FunctionArgList Args; - Args.push_back(&BufferArg); - Args.push_back(&IdxArg); - Args.push_back(&ReduceListArg); - - const CGFunctionInfo &CGFI = - CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); - auto *Fn = llvm::Function::Create( - CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, - "_omp_reduction_global_to_list_copy_func", &CGM.getModule()); - CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); - Fn->setDoesNotRecurse(); - CodeGenFunction CGF(CGM); - CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); - - CGBuilderTy &Bld = CGF.Builder; - - Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); - Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); - Address LocalReduceList( - Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, - C.VoidPtrTy, Loc), - ElemTy->getPointerTo()), - ElemTy, CGF.getPointerAlign()); - QualType StaticTy = C.getRecordType(TeamReductionRec); - llvm::Type *LLVMReductionsBufferTy = - CGM.getTypes().ConvertTypeForMem(StaticTy); - llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), - LLVMReductionsBufferTy->getPointerTo()); - - llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), - /*Volatile=*/false, C.IntTy, - Loc)}; - unsigned Idx = 0; - for (const Expr *Private : Privates) { - // Reduce element = LocalReduceList[i] - Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); - llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( - ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation()); - // elemptr = ((CopyType*)(elemptrptr)) + I - ElemTy = CGF.ConvertTypeForMem(Private->getType()); - ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - ElemPtrPtr, ElemTy->getPointerTo()); - Address ElemPtr = - Address(ElemPtrPtr, ElemTy, C.getTypeAlignInChars(Private->getType())); - const ValueDecl *VD = cast<DeclRefExpr>(Private)->getDecl(); - // Global = Buffer.VD[Idx]; - const FieldDecl *FD = VarFieldMap.lookup(VD); - llvm::Value *BufferPtr = - Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); - LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); - Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.getPointer(), - CGF.ConvertTypeForMem(Private->getType()), - GlobAddr.getAlignment())); - switch (CGF.getEvaluationKind(Private->getType())) { - case TEK_Scalar: { - llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc); - CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(), - LValueBaseInfo(AlignmentSource::Type), - TBAAAccessInfo()); - break; - } - case TEK_Complex: { - CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc); - CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()), - /*isInit=*/false); - break; - } - case TEK_Aggregate: - CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()), - GlobLVal, Private->getType(), - AggValueSlot::DoesNotOverlap); - break; - } - ++Idx; - } - - CGF.FinishFunction(); - return Fn; -} - -/// This function emits a helper that reduces all the reduction variables from -/// the team into the provided global buffer for the reduction variables. -/// -/// void global_to_list_reduce_func(void *buffer, int Idx, void *reduce_data) -/// void *GlobPtrs[]; -/// GlobPtrs[0] = (void*)&buffer.D0[Idx]; -/// ... -/// GlobPtrs[N] = (void*)&buffer.DN[Idx]; -/// reduce_function(reduce_data, GlobPtrs); -static llvm::Value *emitGlobalToListReduceFunction( - CodeGenModule &CGM, ArrayRef<const Expr *> Privates, - QualType ReductionArrayTy, SourceLocation Loc, - const RecordDecl *TeamReductionRec, - const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> - &VarFieldMap, - llvm::Function *ReduceFn) { - ASTContext &C = CGM.getContext(); - - // Buffer: global reduction buffer. - ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - // Idx: index of the buffer. - ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy, - ImplicitParamKind::Other); - // ReduceList: thread local Reduce list. - ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, - C.VoidPtrTy, ImplicitParamKind::Other); - FunctionArgList Args; - Args.push_back(&BufferArg); - Args.push_back(&IdxArg); - Args.push_back(&ReduceListArg); - - const CGFunctionInfo &CGFI = - CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); - auto *Fn = llvm::Function::Create( - CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, - "_omp_reduction_global_to_list_reduce_func", &CGM.getModule()); - CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); - Fn->setDoesNotRecurse(); - CodeGenFunction CGF(CGM); - CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); - - CGBuilderTy &Bld = CGF.Builder; - - Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); - QualType StaticTy = C.getRecordType(TeamReductionRec); - llvm::Type *LLVMReductionsBufferTy = - CGM.getTypes().ConvertTypeForMem(StaticTy); - llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), - LLVMReductionsBufferTy->getPointerTo()); - - // 1. Build a list of reduction variables. - // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; - Address ReductionList = - CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); - auto IPriv = Privates.begin(); - llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), - /*Volatile=*/false, C.IntTy, - Loc)}; - unsigned Idx = 0; - for (unsigned I = 0, E = Privates.size(); I < E; ++I, ++IPriv, ++Idx) { - Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); - // Global = Buffer.VD[Idx]; - const ValueDecl *VD = cast<DeclRefExpr>(*IPriv)->getDecl(); - const FieldDecl *FD = VarFieldMap.lookup(VD); - llvm::Value *BufferPtr = - Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); - LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); - Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, - C.VoidPtrTy); - if ((*IPriv)->getType()->isVariablyModifiedType()) { - // Store array size. - ++Idx; - Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); - llvm::Value *Size = CGF.Builder.CreateIntCast( - CGF.getVLASize( - CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) - .NumElts, - CGF.SizeTy, /*isSigned=*/false); - CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), - Elem); - } - } - - // Call reduce_function(ReduceList, GlobalReduceList) - llvm::Value *GlobalReduceList = ReductionList.getPointer(); - Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); - llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( - AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc); - CGM.getOpenMPRuntime().emitOutlinedFunctionCall( - CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList}); - CGF.FinishFunction(); - return Fn; -} - /// /// Design of OpenMP reductions on the GPU /// @@ -2805,21 +1660,20 @@ void CGOpenMPRuntimeGPU::emitReduction( return; bool ParallelReduction = isOpenMPParallelDirective(Options.ReductionKind); -#ifndef NDEBUG + bool DistributeReduction = isOpenMPDistributeDirective(Options.ReductionKind); bool TeamsReduction = isOpenMPTeamsDirective(Options.ReductionKind); -#endif + + ASTContext &C = CGM.getContext(); if (Options.SimpleReduction) { assert(!TeamsReduction && !ParallelReduction && "Invalid reduction selection in emitReduction."); + (void)ParallelReduction; CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, ReductionOps, Options); return; } - assert((TeamsReduction || ParallelReduction) && - "Invalid reduction selection in emitReduction."); - llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> VarFieldMap; llvm::SmallVector<const ValueDecl *, 4> PrivatesReductions(Privates.size()); int Cnt = 0; @@ -2827,145 +1681,85 @@ void CGOpenMPRuntimeGPU::emitReduction( PrivatesReductions[Cnt] = cast<DeclRefExpr>(DRE)->getDecl(); ++Cnt; } - - ASTContext &C = CGM.getContext(); const RecordDecl *ReductionRec = ::buildRecordForGlobalizedVars( CGM.getContext(), PrivatesReductions, std::nullopt, VarFieldMap, 1); - // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList), - // RedList, shuffle_reduce_func, interwarp_copy_func); - // or - // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>); - llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); - - llvm::Value *Res; - // 1. Build a list of reduction variables. - // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; - auto Size = RHSExprs.size(); - for (const Expr *E : Privates) { - if (E->getType()->isVariablyModifiedType()) - // Reserve place for array size. - ++Size; - } - llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size); - QualType ReductionArrayTy = C.getConstantArrayType( - C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, - /*IndexTypeQuals=*/0); - Address ReductionList = - CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); - auto IPriv = Privates.begin(); - unsigned Idx = 0; - for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) { - Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); - CGF.Builder.CreateStore( - CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), - Elem); - if ((*IPriv)->getType()->isVariablyModifiedType()) { - // Store array size. - ++Idx; - Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); - llvm::Value *Size = CGF.Builder.CreateIntCast( - CGF.getVLASize( - CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) - .NumElts, - CGF.SizeTy, /*isSigned=*/false); - CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), - Elem); - } - } - - llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - ReductionList.getPointer(), CGF.VoidPtrTy); - llvm::Function *ReductionFn = emitReductionFunction( - CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), - Privates, LHSExprs, RHSExprs, ReductionOps); - llvm::Value *ReductionDataSize = - CGF.getTypeSize(C.getRecordType(ReductionRec)); - ReductionDataSize = - CGF.Builder.CreateSExtOrTrunc(ReductionDataSize, CGF.Int64Ty); - llvm::Function *ShuffleAndReduceFn = emitShuffleAndReduceFunction( - CGM, Privates, ReductionArrayTy, ReductionFn, Loc); - llvm::Value *InterWarpCopyFn = - emitInterWarpCopyFunction(CGM, Privates, ReductionArrayTy, Loc); - - if (ParallelReduction) { - llvm::Value *Args[] = {RTLoc, ReductionDataSize, RL, ShuffleAndReduceFn, - InterWarpCopyFn}; - - Res = CGF.EmitRuntimeCall( - OMPBuilder.getOrCreateRuntimeFunction( - CGM.getModule(), OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2), - Args); - } else { - assert(TeamsReduction && "expected teams reduction."); + if (TeamsReduction) TeamsReductions.push_back(ReductionRec); - auto *KernelTeamsReductionPtr = CGF.EmitRuntimeCall( - OMPBuilder.getOrCreateRuntimeFunction( - CGM.getModule(), OMPRTL___kmpc_reduction_get_fixed_buffer), - {}, "_openmp_teams_reductions_buffer_$_$ptr"); - llvm::Value *GlobalToBufferCpyFn = ::emitListToGlobalCopyFunction( - CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap); - llvm::Value *GlobalToBufferRedFn = ::emitListToGlobalReduceFunction( - CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap, - ReductionFn); - llvm::Value *BufferToGlobalCpyFn = ::emitGlobalToListCopyFunction( - CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap); - llvm::Value *BufferToGlobalRedFn = ::emitGlobalToListReduceFunction( - CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap, - ReductionFn); - llvm::Value *Args[] = { - RTLoc, - KernelTeamsReductionPtr, - CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum), - ReductionDataSize, - RL, - ShuffleAndReduceFn, - InterWarpCopyFn, - GlobalToBufferCpyFn, - GlobalToBufferRedFn, - BufferToGlobalCpyFn, - BufferToGlobalRedFn}; - - Res = CGF.EmitRuntimeCall( - OMPBuilder.getOrCreateRuntimeFunction( - CGM.getModule(), OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2), - Args); - } + // Source location for the ident struct + llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); - // 5. Build if (res == 1) - llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); - llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); - llvm::Value *Cond = CGF.Builder.CreateICmpEQ( - Res, llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1)); - CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); - - // 6. Build then branch: where we have reduced values in the master - // thread in each team. - // __kmpc_end_reduce{_nowait}(<gtid>); - // break; - CGF.EmitBlock(ThenBB); - - // Add emission of __kmpc_end_reduce{_nowait}(<gtid>); - auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps, - this](CodeGenFunction &CGF, PrePostActionTy &Action) { - auto IPriv = Privates.begin(); - auto ILHS = LHSExprs.begin(); - auto IRHS = RHSExprs.begin(); - for (const Expr *E : ReductionOps) { - emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), - cast<DeclRefExpr>(*IRHS)); - ++IPriv; - ++ILHS; - ++IRHS; + using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; + InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(), + CGF.AllocaInsertPt->getIterator()); + InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(), + CGF.Builder.GetInsertPoint()); + llvm::OpenMPIRBuilder::LocationDescription OmpLoc( + CodeGenIP, CGF.SourceLocToDebugLoc(Loc)); + llvm::SmallVector<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos; + + CodeGenFunction::OMPPrivateScope Scope(CGF); + unsigned Idx = 0; + for (const Expr *Private : Privates) { + llvm::Type *ElementType; + llvm::Value *Variable; + llvm::Value *PrivateVariable; + llvm::OpenMPIRBuilder::ReductionGenAtomicCBTy AtomicReductionGen = nullptr; + ElementType = CGF.ConvertTypeForMem(Private->getType()); + const auto *RHSVar = + cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[Idx])->getDecl()); + PrivateVariable = CGF.GetAddrOfLocalVar(RHSVar).emitRawPointer(CGF); + const auto *LHSVar = + cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[Idx])->getDecl()); + Variable = CGF.GetAddrOfLocalVar(LHSVar).emitRawPointer(CGF); + llvm::OpenMPIRBuilder::EvalKind EvalKind; + switch (CGF.getEvaluationKind(Private->getType())) { + case TEK_Scalar: + EvalKind = llvm::OpenMPIRBuilder::EvalKind::Scalar; + break; + case TEK_Complex: + EvalKind = llvm::OpenMPIRBuilder::EvalKind::Complex; + break; + case TEK_Aggregate: + EvalKind = llvm::OpenMPIRBuilder::EvalKind::Aggregate; + break; } - }; - RegionCodeGenTy RCG(CodeGen); - RCG(CGF); - // There is no need to emit line number for unconditional branch. - (void)ApplyDebugLocation::CreateEmpty(CGF); - CGF.EmitBlock(ExitBB, /*IsFinished=*/true); + auto ReductionGen = [&](InsertPointTy CodeGenIP, unsigned I, + llvm::Value **LHSPtr, llvm::Value **RHSPtr, + llvm::Function *NewFunc) { + CGF.Builder.restoreIP(CodeGenIP); + auto *CurFn = CGF.CurFn; + CGF.CurFn = NewFunc; + + *LHSPtr = CGF.GetAddrOfLocalVar( + cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl())) + .emitRawPointer(CGF); + *RHSPtr = CGF.GetAddrOfLocalVar( + cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl())) + .emitRawPointer(CGF); + + emitSingleReductionCombiner(CGF, ReductionOps[I], Privates[I], + cast<DeclRefExpr>(LHSExprs[I]), + cast<DeclRefExpr>(RHSExprs[I])); + + CGF.CurFn = CurFn; + + return InsertPointTy(CGF.Builder.GetInsertBlock(), + CGF.Builder.GetInsertPoint()); + }; + ReductionInfos.emplace_back(llvm::OpenMPIRBuilder::ReductionInfo( + ElementType, Variable, PrivateVariable, EvalKind, + /*ReductionGen=*/nullptr, ReductionGen, AtomicReductionGen)); + Idx++; + } + + CGF.Builder.restoreIP(OMPBuilder.createReductionsGPU( + OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, false, TeamsReduction, + DistributeReduction, llvm::OpenMPIRBuilder::ReductionGenCBKind::Clang, + CGF.getTarget().getGridValue(), C.getLangOpts().OpenMPCUDAReductionBufNum, + RTLoc)); + return; } const VarDecl * @@ -3106,15 +1900,15 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper( // Get the array of arguments. SmallVector<llvm::Value *, 8> Args; - Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); - Args.emplace_back(ZeroAddr.getPointer()); + Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF)); + Args.emplace_back(ZeroAddr.emitRawPointer(CGF)); CGBuilderTy &Bld = CGF.Builder; auto CI = CS.capture_begin(); // Use global memory for data sharing. // Handle passing of global args to workers. - Address GlobalArgs = + RawAddress GlobalArgs = CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer(); llvm::Value *DataSharingArgs[] = {GlobalArgsPtr}; @@ -3399,8 +2193,8 @@ void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas( if (VD->getType().getCanonicalType()->isReferenceType()) VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType().getCanonicalType()) - .getAddress(CGF); - CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); + .getAddress(); + CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal); } } } @@ -3434,106 +2228,112 @@ bool CGOpenMPRuntimeGPU::hasAllocateAttributeForGlobalVar(const VarDecl *VD, return false; } -// Get current CudaArch and ignore any unknown values -static CudaArch getCudaArch(CodeGenModule &CGM) { +// Get current OffloadArch and ignore any unknown values +static OffloadArch getOffloadArch(CodeGenModule &CGM) { if (!CGM.getTarget().hasFeature("ptx")) - return CudaArch::UNKNOWN; + return OffloadArch::UNKNOWN; for (const auto &Feature : CGM.getTarget().getTargetOpts().FeatureMap) { if (Feature.getValue()) { - CudaArch Arch = StringToCudaArch(Feature.getKey()); - if (Arch != CudaArch::UNKNOWN) + OffloadArch Arch = StringToOffloadArch(Feature.getKey()); + if (Arch != OffloadArch::UNKNOWN) return Arch; } } - return CudaArch::UNKNOWN; + return OffloadArch::UNKNOWN; } /// Check to see if target architecture supports unified addressing which is /// a restriction for OpenMP requires clause "unified_shared_memory". -void CGOpenMPRuntimeGPU::processRequiresDirective( - const OMPRequiresDecl *D) { +void CGOpenMPRuntimeGPU::processRequiresDirective(const OMPRequiresDecl *D) { for (const OMPClause *Clause : D->clauselists()) { if (Clause->getClauseKind() == OMPC_unified_shared_memory) { - CudaArch Arch = getCudaArch(CGM); + OffloadArch Arch = getOffloadArch(CGM); switch (Arch) { - case CudaArch::SM_20: - case CudaArch::SM_21: - case CudaArch::SM_30: - case CudaArch::SM_32: - case CudaArch::SM_35: - case CudaArch::SM_37: - case CudaArch::SM_50: - case CudaArch::SM_52: - case CudaArch::SM_53: { + case OffloadArch::SM_20: + case OffloadArch::SM_21: + case OffloadArch::SM_30: + case OffloadArch::SM_32_: + case OffloadArch::SM_35: + case OffloadArch::SM_37: + case OffloadArch::SM_50: + case OffloadArch::SM_52: + case OffloadArch::SM_53: { SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); - Out << "Target architecture " << CudaArchToString(Arch) + Out << "Target architecture " << OffloadArchToString(Arch) << " does not support unified addressing"; CGM.Error(Clause->getBeginLoc(), Out.str()); return; } - case CudaArch::SM_60: - case CudaArch::SM_61: - case CudaArch::SM_62: - case CudaArch::SM_70: - case CudaArch::SM_72: - case CudaArch::SM_75: - case CudaArch::SM_80: - case CudaArch::SM_86: - case CudaArch::SM_87: - case CudaArch::SM_89: - case CudaArch::SM_90: - case CudaArch::SM_90a: - case CudaArch::GFX600: - case CudaArch::GFX601: - case CudaArch::GFX602: - case CudaArch::GFX700: - case CudaArch::GFX701: - case CudaArch::GFX702: - case CudaArch::GFX703: - case CudaArch::GFX704: - case CudaArch::GFX705: - case CudaArch::GFX801: - case CudaArch::GFX802: - case CudaArch::GFX803: - case CudaArch::GFX805: - case CudaArch::GFX810: - case CudaArch::GFX900: - case CudaArch::GFX902: - case CudaArch::GFX904: - case CudaArch::GFX906: - case CudaArch::GFX908: - case CudaArch::GFX909: - case CudaArch::GFX90a: - case CudaArch::GFX90c: - case CudaArch::GFX940: - case CudaArch::GFX941: - case CudaArch::GFX942: - case CudaArch::GFX1010: - case CudaArch::GFX1011: - case CudaArch::GFX1012: - case CudaArch::GFX1013: - case CudaArch::GFX1030: - case CudaArch::GFX1031: - case CudaArch::GFX1032: - case CudaArch::GFX1033: - case CudaArch::GFX1034: - case CudaArch::GFX1035: - case CudaArch::GFX1036: - case CudaArch::GFX1100: - case CudaArch::GFX1101: - case CudaArch::GFX1102: - case CudaArch::GFX1103: - case CudaArch::GFX1150: - case CudaArch::GFX1151: - case CudaArch::GFX1200: - case CudaArch::GFX1201: - case CudaArch::Generic: - case CudaArch::UNUSED: - case CudaArch::UNKNOWN: + case OffloadArch::SM_60: + case OffloadArch::SM_61: + case OffloadArch::SM_62: + case OffloadArch::SM_70: + case OffloadArch::SM_72: + case OffloadArch::SM_75: + case OffloadArch::SM_80: + case OffloadArch::SM_86: + case OffloadArch::SM_87: + case OffloadArch::SM_89: + case OffloadArch::SM_90: + case OffloadArch::SM_90a: + case OffloadArch::GFX600: + case OffloadArch::GFX601: + case OffloadArch::GFX602: + case OffloadArch::GFX700: + case OffloadArch::GFX701: + case OffloadArch::GFX702: + case OffloadArch::GFX703: + case OffloadArch::GFX704: + case OffloadArch::GFX705: + case OffloadArch::GFX801: + case OffloadArch::GFX802: + case OffloadArch::GFX803: + case OffloadArch::GFX805: + case OffloadArch::GFX810: + case OffloadArch::GFX9_GENERIC: + case OffloadArch::GFX900: + case OffloadArch::GFX902: + case OffloadArch::GFX904: + case OffloadArch::GFX906: + case OffloadArch::GFX908: + case OffloadArch::GFX909: + case OffloadArch::GFX90a: + case OffloadArch::GFX90c: + case OffloadArch::GFX940: + case OffloadArch::GFX941: + case OffloadArch::GFX942: + case OffloadArch::GFX10_1_GENERIC: + case OffloadArch::GFX1010: + case OffloadArch::GFX1011: + case OffloadArch::GFX1012: + case OffloadArch::GFX1013: + case OffloadArch::GFX10_3_GENERIC: + case OffloadArch::GFX1030: + case OffloadArch::GFX1031: + case OffloadArch::GFX1032: + case OffloadArch::GFX1033: + case OffloadArch::GFX1034: + case OffloadArch::GFX1035: + case OffloadArch::GFX1036: + case OffloadArch::GFX11_GENERIC: + case OffloadArch::GFX1100: + case OffloadArch::GFX1101: + case OffloadArch::GFX1102: + case OffloadArch::GFX1103: + case OffloadArch::GFX1150: + case OffloadArch::GFX1151: + case OffloadArch::GFX1152: + case OffloadArch::GFX12_GENERIC: + case OffloadArch::GFX1200: + case OffloadArch::GFX1201: + case OffloadArch::AMDGCNSPIRV: + case OffloadArch::Generic: + case OffloadArch::UNUSED: + case OffloadArch::UNKNOWN: break; - case CudaArch::LAST: - llvm_unreachable("Unexpected Cuda arch."); + case OffloadArch::LAST: + llvm_unreachable("Unexpected GPU arch."); } } } @@ -3560,10 +2360,3 @@ llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) { CGM.getModule(), OMPRTL___kmpc_get_hardware_thread_id_in_block), Args); } - -llvm::Value *CGOpenMPRuntimeGPU::getGPUWarpSize(CodeGenFunction &CGF) { - ArrayRef<llvm::Value *> Args{}; - return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( - CGM.getModule(), OMPRTL___kmpc_get_warp_size), - Args); -} diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h index 141436f26230..4d586ec972f8 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h @@ -150,9 +150,6 @@ public: CodeGenFunction &CGF, const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) override; - /// Get the GPU warp size. - llvm::Value *getGPUWarpSize(CodeGenFunction &CGF); - /// Get the id of the current thread on the GPU. llvm::Value *getGPUThreadID(CodeGenFunction &CGF); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp new file mode 100644 index 000000000000..0c63b9d6bb7e --- /dev/null +++ b/contrib/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp @@ -0,0 +1,621 @@ +//===--- CGPointerAuth.cpp - IR generation for pointer authentication -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains common routines relating to the emission of +// pointer authentication operations. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenFunction.h" +#include "CodeGenModule.h" +#include "clang/CodeGen/CodeGenABITypes.h" +#include "clang/CodeGen/ConstantInitBuilder.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/Support/SipHash.h" + +using namespace clang; +using namespace CodeGen; + +/// Given a pointer-authentication schema, return a concrete "other" +/// discriminator for it. +llvm::ConstantInt *CodeGenModule::getPointerAuthOtherDiscriminator( + const PointerAuthSchema &Schema, GlobalDecl Decl, QualType Type) { + switch (Schema.getOtherDiscrimination()) { + case PointerAuthSchema::Discrimination::None: + return nullptr; + + case PointerAuthSchema::Discrimination::Type: + assert(!Type.isNull() && "type not provided for type-discriminated schema"); + return llvm::ConstantInt::get( + IntPtrTy, getContext().getPointerAuthTypeDiscriminator(Type)); + + case PointerAuthSchema::Discrimination::Decl: + assert(Decl.getDecl() && + "declaration not provided for decl-discriminated schema"); + return llvm::ConstantInt::get(IntPtrTy, + getPointerAuthDeclDiscriminator(Decl)); + + case PointerAuthSchema::Discrimination::Constant: + return llvm::ConstantInt::get(IntPtrTy, Schema.getConstantDiscrimination()); + } + llvm_unreachable("bad discrimination kind"); +} + +uint16_t CodeGen::getPointerAuthTypeDiscriminator(CodeGenModule &CGM, + QualType FunctionType) { + return CGM.getContext().getPointerAuthTypeDiscriminator(FunctionType); +} + +uint16_t CodeGen::getPointerAuthDeclDiscriminator(CodeGenModule &CGM, + GlobalDecl Declaration) { + return CGM.getPointerAuthDeclDiscriminator(Declaration); +} + +/// Return the "other" decl-specific discriminator for the given decl. +uint16_t +CodeGenModule::getPointerAuthDeclDiscriminator(GlobalDecl Declaration) { + uint16_t &EntityHash = PtrAuthDiscriminatorHashes[Declaration]; + + if (EntityHash == 0) { + StringRef Name = getMangledName(Declaration); + EntityHash = llvm::getPointerAuthStableSipHash(Name); + } + + return EntityHash; +} + +/// Return the abstract pointer authentication schema for a pointer to the given +/// function type. +CGPointerAuthInfo CodeGenModule::getFunctionPointerAuthInfo(QualType T) { + const auto &Schema = getCodeGenOpts().PointerAuth.FunctionPointers; + if (!Schema) + return CGPointerAuthInfo(); + + assert(!Schema.isAddressDiscriminated() && + "function pointers cannot use address-specific discrimination"); + + llvm::Constant *Discriminator = nullptr; + if (T->isFunctionPointerType() || T->isFunctionReferenceType()) + T = T->getPointeeType(); + if (T->isFunctionType()) + Discriminator = getPointerAuthOtherDiscriminator(Schema, GlobalDecl(), T); + + return CGPointerAuthInfo(Schema.getKey(), Schema.getAuthenticationMode(), + /*IsaPointer=*/false, /*AuthenticatesNull=*/false, + Discriminator); +} + +llvm::Value * +CodeGenFunction::EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress, + llvm::Value *Discriminator) { + StorageAddress = Builder.CreatePtrToInt(StorageAddress, IntPtrTy); + auto Intrinsic = CGM.getIntrinsic(llvm::Intrinsic::ptrauth_blend); + return Builder.CreateCall(Intrinsic, {StorageAddress, Discriminator}); +} + +/// Emit the concrete pointer authentication informaton for the +/// given authentication schema. +CGPointerAuthInfo CodeGenFunction::EmitPointerAuthInfo( + const PointerAuthSchema &Schema, llvm::Value *StorageAddress, + GlobalDecl SchemaDecl, QualType SchemaType) { + if (!Schema) + return CGPointerAuthInfo(); + + llvm::Value *Discriminator = + CGM.getPointerAuthOtherDiscriminator(Schema, SchemaDecl, SchemaType); + + if (Schema.isAddressDiscriminated()) { + assert(StorageAddress && + "address not provided for address-discriminated schema"); + + if (Discriminator) + Discriminator = + EmitPointerAuthBlendDiscriminator(StorageAddress, Discriminator); + else + Discriminator = Builder.CreatePtrToInt(StorageAddress, IntPtrTy); + } + + return CGPointerAuthInfo(Schema.getKey(), Schema.getAuthenticationMode(), + Schema.isIsaPointer(), + Schema.authenticatesNullValues(), Discriminator); +} + +/// Return the natural pointer authentication for values of the given +/// pointee type. +static CGPointerAuthInfo +getPointerAuthInfoForPointeeType(CodeGenModule &CGM, QualType PointeeType) { + if (PointeeType.isNull()) + return CGPointerAuthInfo(); + + // Function pointers use the function-pointer schema by default. + if (PointeeType->isFunctionType()) + return CGM.getFunctionPointerAuthInfo(PointeeType); + + // Normal data pointers never use direct pointer authentication by default. + return CGPointerAuthInfo(); +} + +CGPointerAuthInfo CodeGenModule::getPointerAuthInfoForPointeeType(QualType T) { + return ::getPointerAuthInfoForPointeeType(*this, T); +} + +/// Return the natural pointer authentication for values of the given +/// pointer type. +static CGPointerAuthInfo getPointerAuthInfoForType(CodeGenModule &CGM, + QualType PointerType) { + assert(PointerType->isSignableType()); + + // Block pointers are currently not signed. + if (PointerType->isBlockPointerType()) + return CGPointerAuthInfo(); + + auto PointeeType = PointerType->getPointeeType(); + + if (PointeeType.isNull()) + return CGPointerAuthInfo(); + + return ::getPointerAuthInfoForPointeeType(CGM, PointeeType); +} + +CGPointerAuthInfo CodeGenModule::getPointerAuthInfoForType(QualType T) { + return ::getPointerAuthInfoForType(*this, T); +} + +static bool isZeroConstant(const llvm::Value *Value) { + if (const auto *CI = dyn_cast<llvm::ConstantInt>(Value)) + return CI->isZero(); + return false; +} + +static bool equalAuthPolicies(const CGPointerAuthInfo &Left, + const CGPointerAuthInfo &Right) { + assert((Left.isSigned() || Right.isSigned()) && + "shouldn't be called if neither is signed"); + if (Left.isSigned() != Right.isSigned()) + return false; + return Left.getKey() == Right.getKey() && + Left.getAuthenticationMode() == Right.getAuthenticationMode(); +} + +// Return the discriminator or return zero if the discriminator is null. +static llvm::Value *getDiscriminatorOrZero(const CGPointerAuthInfo &Info, + CGBuilderTy &Builder) { + llvm::Value *Discriminator = Info.getDiscriminator(); + return Discriminator ? Discriminator : Builder.getSize(0); +} + +llvm::Value * +CodeGenFunction::emitPointerAuthResignCall(llvm::Value *Value, + const CGPointerAuthInfo &CurAuth, + const CGPointerAuthInfo &NewAuth) { + assert(CurAuth && NewAuth); + + if (CurAuth.getAuthenticationMode() != + PointerAuthenticationMode::SignAndAuth || + NewAuth.getAuthenticationMode() != + PointerAuthenticationMode::SignAndAuth) { + llvm::Value *AuthedValue = EmitPointerAuthAuth(CurAuth, Value); + return EmitPointerAuthSign(NewAuth, AuthedValue); + } + // Convert the pointer to intptr_t before signing it. + auto *OrigType = Value->getType(); + Value = Builder.CreatePtrToInt(Value, IntPtrTy); + + auto *CurKey = Builder.getInt32(CurAuth.getKey()); + auto *NewKey = Builder.getInt32(NewAuth.getKey()); + + llvm::Value *CurDiscriminator = getDiscriminatorOrZero(CurAuth, Builder); + llvm::Value *NewDiscriminator = getDiscriminatorOrZero(NewAuth, Builder); + + // call i64 @llvm.ptrauth.resign(i64 %pointer, + // i32 %curKey, i64 %curDiscriminator, + // i32 %newKey, i64 %newDiscriminator) + auto *Intrinsic = CGM.getIntrinsic(llvm::Intrinsic::ptrauth_resign); + Value = EmitRuntimeCall( + Intrinsic, {Value, CurKey, CurDiscriminator, NewKey, NewDiscriminator}); + + // Convert back to the original type. + Value = Builder.CreateIntToPtr(Value, OrigType); + return Value; +} + +llvm::Value *CodeGenFunction::emitPointerAuthResign( + llvm::Value *Value, QualType Type, const CGPointerAuthInfo &CurAuthInfo, + const CGPointerAuthInfo &NewAuthInfo, bool IsKnownNonNull) { + // Fast path: if neither schema wants a signature, we're done. + if (!CurAuthInfo && !NewAuthInfo) + return Value; + + llvm::Value *Null = nullptr; + // If the value is obviously null, we're done. + if (auto *PointerValue = dyn_cast<llvm::PointerType>(Value->getType())) { + Null = CGM.getNullPointer(PointerValue, Type); + } else { + assert(Value->getType()->isIntegerTy()); + Null = llvm::ConstantInt::get(IntPtrTy, 0); + } + if (Value == Null) + return Value; + + // If both schemas sign the same way, we're done. + if (equalAuthPolicies(CurAuthInfo, NewAuthInfo)) { + const llvm::Value *CurD = CurAuthInfo.getDiscriminator(); + const llvm::Value *NewD = NewAuthInfo.getDiscriminator(); + if (CurD == NewD) + return Value; + + if ((CurD == nullptr && isZeroConstant(NewD)) || + (NewD == nullptr && isZeroConstant(CurD))) + return Value; + } + + llvm::BasicBlock *InitBB = Builder.GetInsertBlock(); + llvm::BasicBlock *ResignBB = nullptr, *ContBB = nullptr; + + // Null pointers have to be mapped to null, and the ptrauth_resign + // intrinsic doesn't do that. + if (!IsKnownNonNull && !llvm::isKnownNonZero(Value, CGM.getDataLayout())) { + ContBB = createBasicBlock("resign.cont"); + ResignBB = createBasicBlock("resign.nonnull"); + + auto *IsNonNull = Builder.CreateICmpNE(Value, Null); + Builder.CreateCondBr(IsNonNull, ResignBB, ContBB); + EmitBlock(ResignBB); + } + + // Perform the auth/sign/resign operation. + if (!NewAuthInfo) + Value = EmitPointerAuthAuth(CurAuthInfo, Value); + else if (!CurAuthInfo) + Value = EmitPointerAuthSign(NewAuthInfo, Value); + else + Value = emitPointerAuthResignCall(Value, CurAuthInfo, NewAuthInfo); + + // Clean up with a phi if we branched before. + if (ContBB) { + EmitBlock(ContBB); + auto *Phi = Builder.CreatePHI(Value->getType(), 2); + Phi->addIncoming(Null, InitBB); + Phi->addIncoming(Value, ResignBB); + Value = Phi; + } + + return Value; +} + +llvm::Constant * +CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key, + llvm::Constant *StorageAddress, + llvm::ConstantInt *OtherDiscriminator) { + llvm::Constant *AddressDiscriminator; + if (StorageAddress) { + assert(StorageAddress->getType() == UnqualPtrTy); + AddressDiscriminator = StorageAddress; + } else { + AddressDiscriminator = llvm::Constant::getNullValue(UnqualPtrTy); + } + + llvm::ConstantInt *IntegerDiscriminator; + if (OtherDiscriminator) { + assert(OtherDiscriminator->getType() == Int64Ty); + IntegerDiscriminator = OtherDiscriminator; + } else { + IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0); + } + + return llvm::ConstantPtrAuth::get(Pointer, + llvm::ConstantInt::get(Int32Ty, Key), + IntegerDiscriminator, AddressDiscriminator); +} + +/// Does a given PointerAuthScheme require us to sign a value +bool CodeGenModule::shouldSignPointer(const PointerAuthSchema &Schema) { + auto AuthenticationMode = Schema.getAuthenticationMode(); + return AuthenticationMode == PointerAuthenticationMode::SignAndStrip || + AuthenticationMode == PointerAuthenticationMode::SignAndAuth; +} + +/// Sign a constant pointer using the given scheme, producing a constant +/// with the same IR type. +llvm::Constant *CodeGenModule::getConstantSignedPointer( + llvm::Constant *Pointer, const PointerAuthSchema &Schema, + llvm::Constant *StorageAddress, GlobalDecl SchemaDecl, + QualType SchemaType) { + assert(shouldSignPointer(Schema)); + llvm::ConstantInt *OtherDiscriminator = + getPointerAuthOtherDiscriminator(Schema, SchemaDecl, SchemaType); + + return getConstantSignedPointer(Pointer, Schema.getKey(), StorageAddress, + OtherDiscriminator); +} + +/// If applicable, sign a given constant function pointer with the ABI rules for +/// functionType. +llvm::Constant *CodeGenModule::getFunctionPointer(llvm::Constant *Pointer, + QualType FunctionType) { + assert(FunctionType->isFunctionType() || + FunctionType->isFunctionReferenceType() || + FunctionType->isFunctionPointerType()); + + if (auto PointerAuth = getFunctionPointerAuthInfo(FunctionType)) + return getConstantSignedPointer( + Pointer, PointerAuth.getKey(), /*StorageAddress=*/nullptr, + cast_or_null<llvm::ConstantInt>(PointerAuth.getDiscriminator())); + + return Pointer; +} + +llvm::Constant *CodeGenModule::getFunctionPointer(GlobalDecl GD, + llvm::Type *Ty) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + QualType FuncType = FD->getType(); + + // Annoyingly, K&R functions have prototypes in the clang AST, but + // expressions referring to them are unprototyped. + if (!FD->hasPrototype()) + if (const auto *Proto = FuncType->getAs<FunctionProtoType>()) + FuncType = Context.getFunctionNoProtoType(Proto->getReturnType(), + Proto->getExtInfo()); + + return getFunctionPointer(getRawFunctionPointer(GD, Ty), FuncType); +} + +CGPointerAuthInfo CodeGenModule::getMemberFunctionPointerAuthInfo(QualType FT) { + assert(FT->getAs<MemberPointerType>() && "MemberPointerType expected"); + const auto &Schema = getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers; + if (!Schema) + return CGPointerAuthInfo(); + + assert(!Schema.isAddressDiscriminated() && + "function pointers cannot use address-specific discrimination"); + + llvm::ConstantInt *Discriminator = + getPointerAuthOtherDiscriminator(Schema, GlobalDecl(), FT); + return CGPointerAuthInfo(Schema.getKey(), Schema.getAuthenticationMode(), + /* IsIsaPointer */ false, + /* AuthenticatesNullValues */ false, Discriminator); +} + +llvm::Constant *CodeGenModule::getMemberFunctionPointer(llvm::Constant *Pointer, + QualType FT) { + if (CGPointerAuthInfo PointerAuth = getMemberFunctionPointerAuthInfo(FT)) + return getConstantSignedPointer( + Pointer, PointerAuth.getKey(), nullptr, + cast_or_null<llvm::ConstantInt>(PointerAuth.getDiscriminator())); + + return Pointer; +} + +llvm::Constant *CodeGenModule::getMemberFunctionPointer(const FunctionDecl *FD, + llvm::Type *Ty) { + QualType FT = FD->getType(); + FT = getContext().getMemberPointerType( + FT, cast<CXXMethodDecl>(FD)->getParent()->getTypeForDecl()); + return getMemberFunctionPointer(getRawFunctionPointer(FD, Ty), FT); +} + +std::optional<PointerAuthQualifier> +CodeGenModule::computeVTPointerAuthentication(const CXXRecordDecl *ThisClass) { + auto DefaultAuthentication = getCodeGenOpts().PointerAuth.CXXVTablePointers; + if (!DefaultAuthentication) + return std::nullopt; + const CXXRecordDecl *PrimaryBase = + Context.baseForVTableAuthentication(ThisClass); + + unsigned Key = DefaultAuthentication.getKey(); + bool AddressDiscriminated = DefaultAuthentication.isAddressDiscriminated(); + auto DefaultDiscrimination = DefaultAuthentication.getOtherDiscrimination(); + unsigned TypeBasedDiscriminator = + Context.getPointerAuthVTablePointerDiscriminator(PrimaryBase); + unsigned Discriminator; + if (DefaultDiscrimination == PointerAuthSchema::Discrimination::Type) { + Discriminator = TypeBasedDiscriminator; + } else if (DefaultDiscrimination == + PointerAuthSchema::Discrimination::Constant) { + Discriminator = DefaultAuthentication.getConstantDiscrimination(); + } else { + assert(DefaultDiscrimination == PointerAuthSchema::Discrimination::None); + Discriminator = 0; + } + if (auto ExplicitAuthentication = + PrimaryBase->getAttr<VTablePointerAuthenticationAttr>()) { + auto ExplicitAddressDiscrimination = + ExplicitAuthentication->getAddressDiscrimination(); + auto ExplicitDiscriminator = + ExplicitAuthentication->getExtraDiscrimination(); + + unsigned ExplicitKey = ExplicitAuthentication->getKey(); + if (ExplicitKey == VTablePointerAuthenticationAttr::NoKey) + return std::nullopt; + + if (ExplicitKey != VTablePointerAuthenticationAttr::DefaultKey) { + if (ExplicitKey == VTablePointerAuthenticationAttr::ProcessIndependent) + Key = (unsigned)PointerAuthSchema::ARM8_3Key::ASDA; + else { + assert(ExplicitKey == + VTablePointerAuthenticationAttr::ProcessDependent); + Key = (unsigned)PointerAuthSchema::ARM8_3Key::ASDB; + } + } + + if (ExplicitAddressDiscrimination != + VTablePointerAuthenticationAttr::DefaultAddressDiscrimination) + AddressDiscriminated = + ExplicitAddressDiscrimination == + VTablePointerAuthenticationAttr::AddressDiscrimination; + + if (ExplicitDiscriminator == + VTablePointerAuthenticationAttr::TypeDiscrimination) + Discriminator = TypeBasedDiscriminator; + else if (ExplicitDiscriminator == + VTablePointerAuthenticationAttr::CustomDiscrimination) + Discriminator = ExplicitAuthentication->getCustomDiscriminationValue(); + else if (ExplicitDiscriminator == + VTablePointerAuthenticationAttr::NoExtraDiscrimination) + Discriminator = 0; + } + return PointerAuthQualifier::Create(Key, AddressDiscriminated, Discriminator, + PointerAuthenticationMode::SignAndAuth, + /* IsIsaPointer */ false, + /* AuthenticatesNullValues */ false); +} + +std::optional<PointerAuthQualifier> +CodeGenModule::getVTablePointerAuthentication(const CXXRecordDecl *Record) { + if (!Record->getDefinition() || !Record->isPolymorphic()) + return std::nullopt; + + auto Existing = VTablePtrAuthInfos.find(Record); + std::optional<PointerAuthQualifier> Authentication; + if (Existing != VTablePtrAuthInfos.end()) { + Authentication = Existing->getSecond(); + } else { + Authentication = computeVTPointerAuthentication(Record); + VTablePtrAuthInfos.insert(std::make_pair(Record, Authentication)); + } + return Authentication; +} + +std::optional<CGPointerAuthInfo> +CodeGenModule::getVTablePointerAuthInfo(CodeGenFunction *CGF, + const CXXRecordDecl *Record, + llvm::Value *StorageAddress) { + auto Authentication = getVTablePointerAuthentication(Record); + if (!Authentication) + return std::nullopt; + + llvm::Value *Discriminator = nullptr; + if (auto ExtraDiscriminator = Authentication->getExtraDiscriminator()) + Discriminator = llvm::ConstantInt::get(IntPtrTy, ExtraDiscriminator); + + if (Authentication->isAddressDiscriminated()) { + assert(StorageAddress && + "address not provided for address-discriminated schema"); + if (Discriminator) + Discriminator = + CGF->EmitPointerAuthBlendDiscriminator(StorageAddress, Discriminator); + else + Discriminator = CGF->Builder.CreatePtrToInt(StorageAddress, IntPtrTy); + } + + return CGPointerAuthInfo(Authentication->getKey(), + PointerAuthenticationMode::SignAndAuth, + /* IsIsaPointer */ false, + /* AuthenticatesNullValues */ false, Discriminator); +} + +llvm::Value *CodeGenFunction::authPointerToPointerCast(llvm::Value *ResultPtr, + QualType SourceType, + QualType DestType) { + CGPointerAuthInfo CurAuthInfo, NewAuthInfo; + if (SourceType->isSignableType()) + CurAuthInfo = getPointerAuthInfoForType(CGM, SourceType); + + if (DestType->isSignableType()) + NewAuthInfo = getPointerAuthInfoForType(CGM, DestType); + + if (!CurAuthInfo && !NewAuthInfo) + return ResultPtr; + + // If only one side of the cast is a function pointer, then we still need to + // resign to handle casts to/from opaque pointers. + if (!CurAuthInfo && DestType->isFunctionPointerType()) + CurAuthInfo = CGM.getFunctionPointerAuthInfo(SourceType); + + if (!NewAuthInfo && SourceType->isFunctionPointerType()) + NewAuthInfo = CGM.getFunctionPointerAuthInfo(DestType); + + return emitPointerAuthResign(ResultPtr, DestType, CurAuthInfo, NewAuthInfo, + /*IsKnownNonNull=*/false); +} + +Address CodeGenFunction::authPointerToPointerCast(Address Ptr, + QualType SourceType, + QualType DestType) { + CGPointerAuthInfo CurAuthInfo, NewAuthInfo; + if (SourceType->isSignableType()) + CurAuthInfo = getPointerAuthInfoForType(CGM, SourceType); + + if (DestType->isSignableType()) + NewAuthInfo = getPointerAuthInfoForType(CGM, DestType); + + if (!CurAuthInfo && !NewAuthInfo) + return Ptr; + + if (!CurAuthInfo && DestType->isFunctionPointerType()) { + // When casting a non-signed pointer to a function pointer, just set the + // auth info on Ptr to the assumed schema. The pointer will be resigned to + // the effective type when used. + Ptr.setPointerAuthInfo(CGM.getFunctionPointerAuthInfo(SourceType)); + return Ptr; + } + + if (!NewAuthInfo && SourceType->isFunctionPointerType()) { + NewAuthInfo = CGM.getFunctionPointerAuthInfo(DestType); + Ptr = Ptr.getResignedAddress(NewAuthInfo, *this); + Ptr.setPointerAuthInfo(CGPointerAuthInfo()); + return Ptr; + } + + return Ptr; +} + +Address CodeGenFunction::getAsNaturalAddressOf(Address Addr, + QualType PointeeTy) { + CGPointerAuthInfo Info = + PointeeTy.isNull() ? CGPointerAuthInfo() + : CGM.getPointerAuthInfoForPointeeType(PointeeTy); + return Addr.getResignedAddress(Info, *this); +} + +Address Address::getResignedAddress(const CGPointerAuthInfo &NewInfo, + CodeGenFunction &CGF) const { + assert(isValid() && "pointer isn't valid"); + CGPointerAuthInfo CurInfo = getPointerAuthInfo(); + llvm::Value *Val; + + // Nothing to do if neither the current or the new ptrauth info needs signing. + if (!CurInfo.isSigned() && !NewInfo.isSigned()) + return Address(getBasePointer(), getElementType(), getAlignment(), + isKnownNonNull()); + + assert(ElementType && "Effective type has to be set"); + assert(!Offset && "unexpected non-null offset"); + + // If the current and the new ptrauth infos are the same and the offset is + // null, just cast the base pointer to the effective type. + if (CurInfo == NewInfo && !hasOffset()) + Val = getBasePointer(); + else + Val = CGF.emitPointerAuthResign(getBasePointer(), QualType(), CurInfo, + NewInfo, isKnownNonNull()); + + Val = CGF.Builder.CreateBitCast(Val, getType()); + return Address(Val, getElementType(), getAlignment(), NewInfo, + /*Offset=*/nullptr, isKnownNonNull()); +} + +llvm::Value *Address::emitRawPointerSlow(CodeGenFunction &CGF) const { + return CGF.getAsNaturalPointerTo(*this, QualType()); +} + +llvm::Value *LValue::getPointer(CodeGenFunction &CGF) const { + assert(isSimple()); + return emitResignedPointer(getType(), CGF); +} + +llvm::Value *LValue::emitResignedPointer(QualType PointeeTy, + CodeGenFunction &CGF) const { + assert(isSimple()); + return CGF.getAsNaturalAddressOf(Addr, PointeeTy).getBasePointer(); +} + +llvm::Value *LValue::emitRawPointer(CodeGenFunction &CGF) const { + assert(isSimple()); + return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; +} diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGPointerAuthInfo.h b/contrib/llvm-project/clang/lib/CodeGen/CGPointerAuthInfo.h new file mode 100644 index 000000000000..0a0c11fb423f --- /dev/null +++ b/contrib/llvm-project/clang/lib/CodeGen/CGPointerAuthInfo.h @@ -0,0 +1,99 @@ +//===----- CGPointerAuthInfo.h - -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Pointer auth info class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_CODEGEN_CGPOINTERAUTHINFO_H +#define LLVM_CLANG_LIB_CODEGEN_CGPOINTERAUTHINFO_H + +#include "clang/AST/Type.h" +#include "clang/Basic/LangOptions.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" + +namespace clang { +namespace CodeGen { + +class CGPointerAuthInfo { +private: + PointerAuthenticationMode AuthenticationMode : 2; + unsigned IsIsaPointer : 1; + unsigned AuthenticatesNullValues : 1; + unsigned Key : 2; + llvm::Value *Discriminator; + +public: + CGPointerAuthInfo() + : AuthenticationMode(PointerAuthenticationMode::None), + IsIsaPointer(false), AuthenticatesNullValues(false), Key(0), + Discriminator(nullptr) {} + CGPointerAuthInfo(unsigned Key, PointerAuthenticationMode AuthenticationMode, + bool IsIsaPointer, bool AuthenticatesNullValues, + llvm::Value *Discriminator) + : AuthenticationMode(AuthenticationMode), IsIsaPointer(IsIsaPointer), + AuthenticatesNullValues(AuthenticatesNullValues), Key(Key), + Discriminator(Discriminator) { + assert(!Discriminator || Discriminator->getType()->isIntegerTy() || + Discriminator->getType()->isPointerTy()); + } + + explicit operator bool() const { return isSigned(); } + + bool isSigned() const { + return AuthenticationMode != PointerAuthenticationMode::None; + } + + unsigned getKey() const { + assert(isSigned()); + return Key; + } + llvm::Value *getDiscriminator() const { + assert(isSigned()); + return Discriminator; + } + + PointerAuthenticationMode getAuthenticationMode() const { + return AuthenticationMode; + } + + bool isIsaPointer() const { return IsIsaPointer; } + + bool authenticatesNullValues() const { return AuthenticatesNullValues; } + + bool shouldStrip() const { + return AuthenticationMode == PointerAuthenticationMode::Strip || + AuthenticationMode == PointerAuthenticationMode::SignAndStrip; + } + + bool shouldSign() const { + return AuthenticationMode == PointerAuthenticationMode::SignAndStrip || + AuthenticationMode == PointerAuthenticationMode::SignAndAuth; + } + + bool shouldAuth() const { + return AuthenticationMode == PointerAuthenticationMode::SignAndAuth; + } + + friend bool operator!=(const CGPointerAuthInfo &LHS, + const CGPointerAuthInfo &RHS) { + return LHS.Key != RHS.Key || LHS.Discriminator != RHS.Discriminator || + LHS.AuthenticationMode != RHS.AuthenticationMode; + } + + friend bool operator==(const CGPointerAuthInfo &LHS, + const CGPointerAuthInfo &RHS) { + return !(LHS != RHS); + } +}; + +} // end namespace CodeGen +} // end namespace clang + +#endif diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayout.h b/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayout.h index d5ea74922603..44e888c93108 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayout.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayout.h @@ -71,6 +71,7 @@ struct CGBitFieldInfo { unsigned Size : 15; /// Whether the bit-field is signed. + LLVM_PREFERRED_TYPE(bool) unsigned IsSigned : 1; /// The storage size in bits which should be used when accessing this @@ -192,6 +193,10 @@ public: return IsZeroInitializableAsBase; } + bool containsFieldDecl(const FieldDecl *FD) const { + return FieldInfo.count(FD) != 0; + } + /// Return llvm::StructType element number that corresponds to the /// field FD. unsigned getLLVMFieldNo(const FieldDecl *FD) const { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp index 868ef810f3c4..ea44e6f21f3c 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp @@ -10,8 +10,9 @@ // //===----------------------------------------------------------------------===// -#include "CGRecordLayout.h" +#include "ABIInfoImpl.h" #include "CGCXXABI.h" +#include "CGRecordLayout.h" #include "CodeGenTypes.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" @@ -41,14 +42,17 @@ namespace { /// contains enough information to determine where the runs break. Microsoft /// and Itanium follow different rules and use different codepaths. /// * It is desired that, when possible, bitfields use the appropriate iN type -/// when lowered to llvm types. For example unsigned x : 24 gets lowered to +/// when lowered to llvm types. For example unsigned x : 24 gets lowered to /// i24. This isn't always possible because i24 has storage size of 32 bit -/// and if it is possible to use that extra byte of padding we must use -/// [i8 x 3] instead of i24. The function clipTailPadding does this. +/// and if it is possible to use that extra byte of padding we must use [i8 x +/// 3] instead of i24. This is computed when accumulating bitfields in +/// accumulateBitfields. /// C++ examples that require clipping: /// struct { int a : 24; char b; }; // a must be clipped, b goes at offset 3 -/// struct A { int a : 24; }; // a must be clipped because a struct like B -// could exist: struct B : A { char b; }; // b goes at offset 3 +/// struct A { int a : 24; ~A(); }; // a must be clipped because: +/// struct B : A { char b; }; // b goes at offset 3 +/// * The allocation of bitfield access units is described in more detail in +/// CGRecordLowering::accumulateBitFields. /// * Clang ignores 0 sized bitfields and 0 sized bases but *not* zero sized /// fields. The existing asserts suggest that LLVM assumes that *every* field /// has an underlying storage type. Therefore empty structures containing @@ -60,11 +64,7 @@ namespace { /// that the tail padding is not used in the complete class.) However, /// because LLVM reads from the complete type it can generate incorrect code /// if we do not clip the tail padding off of the bitfield in the complete -/// layout. This introduces a somewhat awkward extra unnecessary clip stage. -/// The location of the clip is stored internally as a sentinel of type -/// SCISSOR. If LLVM were updated to read base types (which it probably -/// should because locations of things such as VBases are bogus in the llvm -/// type anyway) then we could eliminate the SCISSOR. +/// layout. /// * Itanium allows nearly empty primary virtual bases. These bases don't get /// get their own storage because they're laid out as part of another base /// or at the beginning of the structure. Determining if a VBase actually @@ -76,7 +76,7 @@ struct CGRecordLowering { // sentinel member type that ensures correct rounding. struct MemberInfo { CharUnits Offset; - enum InfoKind { VFPtr, VBPtr, Field, Base, VBase, Scissor } Kind; + enum InfoKind { VFPtr, VBPtr, Field, Base, VBase } Kind; llvm::Type *Data; union { const FieldDecl *FD; @@ -95,7 +95,7 @@ struct CGRecordLowering { CGRecordLowering(CodeGenTypes &Types, const RecordDecl *D, bool Packed); // Short helper routines. /// Constructs a MemberInfo instance from an offset and llvm::Type *. - MemberInfo StorageInfo(CharUnits Offset, llvm::Type *Data) { + static MemberInfo StorageInfo(CharUnits Offset, llvm::Type *Data) { return MemberInfo(Offset, MemberInfo::Field, Data); } @@ -104,7 +104,7 @@ struct CGRecordLowering { /// fields of the same formal type. We want to emit a layout with /// these discrete storage units instead of combining them into a /// continuous run. - bool isDiscreteBitFieldABI() { + bool isDiscreteBitFieldABI() const { return Context.getTargetInfo().getCXXABI().isMicrosoft() || D->isMsStruct(Context); } @@ -121,22 +121,22 @@ struct CGRecordLowering { /// other bases, which complicates layout in specific ways. /// /// Note specifically that the ms_struct attribute doesn't change this. - bool isOverlappingVBaseABI() { + bool isOverlappingVBaseABI() const { return !Context.getTargetInfo().getCXXABI().isMicrosoft(); } /// Wraps llvm::Type::getIntNTy with some implicit arguments. - llvm::Type *getIntNType(uint64_t NumBits) { + llvm::Type *getIntNType(uint64_t NumBits) const { unsigned AlignedBits = llvm::alignTo(NumBits, Context.getCharWidth()); return llvm::Type::getIntNTy(Types.getLLVMContext(), AlignedBits); } /// Get the LLVM type sized as one character unit. - llvm::Type *getCharType() { + llvm::Type *getCharType() const { return llvm::Type::getIntNTy(Types.getLLVMContext(), Context.getCharWidth()); } /// Gets an llvm type of size NumChars and alignment 1. - llvm::Type *getByteArrayType(CharUnits NumChars) { + llvm::Type *getByteArrayType(CharUnits NumChars) const { assert(!NumChars.isZero() && "Empty byte arrays aren't allowed."); llvm::Type *Type = getCharType(); return NumChars == CharUnits::One() ? Type : @@ -144,7 +144,7 @@ struct CGRecordLowering { } /// Gets the storage type for a field decl and handles storage /// for itanium bitfields that are smaller than their declared type. - llvm::Type *getStorageType(const FieldDecl *FD) { + llvm::Type *getStorageType(const FieldDecl *FD) const { llvm::Type *Type = Types.ConvertTypeForMem(FD->getType()); if (!FD->isBitField()) return Type; if (isDiscreteBitFieldABI()) return Type; @@ -152,29 +152,29 @@ struct CGRecordLowering { (unsigned)Context.toBits(getSize(Type)))); } /// Gets the llvm Basesubobject type from a CXXRecordDecl. - llvm::Type *getStorageType(const CXXRecordDecl *RD) { + llvm::Type *getStorageType(const CXXRecordDecl *RD) const { return Types.getCGRecordLayout(RD).getBaseSubobjectLLVMType(); } - CharUnits bitsToCharUnits(uint64_t BitOffset) { + CharUnits bitsToCharUnits(uint64_t BitOffset) const { return Context.toCharUnitsFromBits(BitOffset); } - CharUnits getSize(llvm::Type *Type) { + CharUnits getSize(llvm::Type *Type) const { return CharUnits::fromQuantity(DataLayout.getTypeAllocSize(Type)); } - CharUnits getAlignment(llvm::Type *Type) { + CharUnits getAlignment(llvm::Type *Type) const { return CharUnits::fromQuantity(DataLayout.getABITypeAlign(Type)); } - bool isZeroInitializable(const FieldDecl *FD) { + bool isZeroInitializable(const FieldDecl *FD) const { return Types.isZeroInitializable(FD->getType()); } - bool isZeroInitializable(const RecordDecl *RD) { + bool isZeroInitializable(const RecordDecl *RD) const { return Types.isZeroInitializable(RD); } void appendPaddingBytes(CharUnits Size) { if (!Size.isZero()) FieldTypes.push_back(getByteArrayType(Size)); } - uint64_t getFieldBitOffset(const FieldDecl *FD) { + uint64_t getFieldBitOffset(const FieldDecl *FD) const { return Layout.getFieldOffset(FD->getFieldIndex()); } // Layout routines. @@ -183,20 +183,22 @@ struct CGRecordLowering { /// Lowers an ASTRecordLayout to a llvm type. void lower(bool NonVirtualBaseType); void lowerUnion(bool isNoUniqueAddress); - void accumulateFields(); - void accumulateBitFields(RecordDecl::field_iterator Field, - RecordDecl::field_iterator FieldEnd); + void accumulateFields(bool isNonVirtualBaseType); + RecordDecl::field_iterator + accumulateBitFields(bool isNonVirtualBaseType, + RecordDecl::field_iterator Field, + RecordDecl::field_iterator FieldEnd); void computeVolatileBitfields(); void accumulateBases(); void accumulateVPtrs(); void accumulateVBases(); /// Recursively searches all of the bases to find out if a vbase is /// not the primary vbase of some base class. - bool hasOwnStorage(const CXXRecordDecl *Decl, const CXXRecordDecl *Query); + bool hasOwnStorage(const CXXRecordDecl *Decl, + const CXXRecordDecl *Query) const; void calculateZeroInit(); - /// Lowers bitfield storage types to I8 arrays for bitfields with tail - /// padding that is or can potentially be used. - void clipTailPadding(); + CharUnits calculateTailClippingOffset(bool isNonVirtualBaseType) const; + void checkBitfieldClipping(bool isNonVirtualBaseType) const; /// Determines if we need a packed llvm struct. void determinePacked(bool NVBaseType); /// Inserts padding everywhere it's needed. @@ -284,7 +286,7 @@ void CGRecordLowering::lower(bool NVBaseType) { computeVolatileBitfields(); return; } - accumulateFields(); + accumulateFields(NVBaseType); // RD implies C++. if (RD) { accumulateVPtrs(); @@ -298,8 +300,8 @@ void CGRecordLowering::lower(bool NVBaseType) { accumulateVBases(); } llvm::stable_sort(Members); + checkBitfieldClipping(NVBaseType); Members.push_back(StorageInfo(Size, getIntNType(8))); - clipTailPadding(); determinePacked(NVBaseType); insertPadding(); Members.pop_back(); @@ -375,16 +377,18 @@ void CGRecordLowering::lowerUnion(bool isNoUniqueAddress) { Packed = true; } -void CGRecordLowering::accumulateFields() { +void CGRecordLowering::accumulateFields(bool isNonVirtualBaseType) { for (RecordDecl::field_iterator Field = D->field_begin(), FieldEnd = D->field_end(); - Field != FieldEnd;) { + Field != FieldEnd;) { if (Field->isBitField()) { - RecordDecl::field_iterator Start = Field; - // Iterate to gather the list of bitfields. - for (++Field; Field != FieldEnd && Field->isBitField(); ++Field); - accumulateBitFields(Start, Field); - } else if (!Field->isZeroSize(Context)) { + Field = accumulateBitFields(isNonVirtualBaseType, Field, FieldEnd); + assert((Field == FieldEnd || !Field->isBitField()) && + "Failed to accumulate all the bitfields"); + } else if (isEmptyFieldForLayout(Context, *Field)) { + // Empty fields have no storage. + ++Field; + } else { // Use base subobject layout for the potentially-overlapping field, // as it is done in RecordLayoutBuilder Members.push_back(MemberInfo( @@ -394,35 +398,37 @@ void CGRecordLowering::accumulateFields() { : getStorageType(*Field), *Field)); ++Field; - } else { - ++Field; } } } -void -CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, +// Create members for bitfields. Field is a bitfield, and FieldEnd is the end +// iterator of the record. Return the first non-bitfield encountered. We need +// to know whether this is the base or complete layout, as virtual bases could +// affect the upper bound of bitfield access unit allocation. +RecordDecl::field_iterator +CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType, + RecordDecl::field_iterator Field, RecordDecl::field_iterator FieldEnd) { - // Run stores the first element of the current run of bitfields. FieldEnd is - // used as a special value to note that we don't have a current run. A - // bitfield run is a contiguous collection of bitfields that can be stored in - // the same storage block. Zero-sized bitfields and bitfields that would - // cross an alignment boundary break a run and start a new one. - RecordDecl::field_iterator Run = FieldEnd; - // Tail is the offset of the first bit off the end of the current run. It's - // used to determine if the ASTRecordLayout is treating these two bitfields as - // contiguous. StartBitOffset is offset of the beginning of the Run. - uint64_t StartBitOffset, Tail = 0; if (isDiscreteBitFieldABI()) { - for (; Field != FieldEnd; ++Field) { - uint64_t BitOffset = getFieldBitOffset(*Field); + // Run stores the first element of the current run of bitfields. FieldEnd is + // used as a special value to note that we don't have a current run. A + // bitfield run is a contiguous collection of bitfields that can be stored + // in the same storage block. Zero-sized bitfields and bitfields that would + // cross an alignment boundary break a run and start a new one. + RecordDecl::field_iterator Run = FieldEnd; + // Tail is the offset of the first bit off the end of the current run. It's + // used to determine if the ASTRecordLayout is treating these two bitfields + // as contiguous. StartBitOffset is offset of the beginning of the Run. + uint64_t StartBitOffset, Tail = 0; + for (; Field != FieldEnd && Field->isBitField(); ++Field) { // Zero-width bitfields end runs. if (Field->isZeroLengthBitField(Context)) { Run = FieldEnd; continue; } - llvm::Type *Type = - Types.ConvertTypeForMem(Field->getType(), /*ForBitField=*/true); + uint64_t BitOffset = getFieldBitOffset(*Field); + llvm::Type *Type = Types.ConvertTypeForMem(Field->getType()); // If we don't have a run yet, or don't live within the previous run's // allocated storage then we allocate some storage and start a new run. if (Run == FieldEnd || BitOffset >= Tail) { @@ -439,82 +445,276 @@ CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, Members.push_back(MemberInfo(bitsToCharUnits(StartBitOffset), MemberInfo::Field, nullptr, *Field)); } - return; + return Field; } - // Check if OffsetInRecord (the size in bits of the current run) is better - // as a single field run. When OffsetInRecord has legal integer width, and - // its bitfield offset is naturally aligned, it is better to make the - // bitfield a separate storage component so as it can be accessed directly - // with lower cost. - auto IsBetterAsSingleFieldRun = [&](uint64_t OffsetInRecord, - uint64_t StartBitOffset) { - if (!Types.getCodeGenOpts().FineGrainedBitfieldAccesses) - return false; - if (OffsetInRecord < 8 || !llvm::isPowerOf2_64(OffsetInRecord) || - !DataLayout.fitsInLegalInteger(OffsetInRecord)) - return false; - // Make sure StartBitOffset is naturally aligned if it is treated as an - // IType integer. - if (StartBitOffset % - Context.toBits(getAlignment(getIntNType(OffsetInRecord))) != - 0) - return false; - return true; - }; + // The SysV ABI can overlap bitfield storage units with both other bitfield + // storage units /and/ other non-bitfield data members. Accessing a sequence + // of bitfields mustn't interfere with adjacent non-bitfields -- they're + // permitted to be accessed in separate threads for instance. + + // We split runs of bit-fields into a sequence of "access units". When we emit + // a load or store of a bit-field, we'll load/store the entire containing + // access unit. As mentioned, the standard requires that these loads and + // stores must not interfere with accesses to other memory locations, and it + // defines the bit-field's memory location as the current run of + // non-zero-width bit-fields. So an access unit must never overlap with + // non-bit-field storage or cross a zero-width bit-field. Otherwise, we're + // free to draw the lines as we see fit. + + // Drawing these lines well can be complicated. LLVM generally can't modify a + // program to access memory that it didn't before, so using very narrow access + // units can prevent the compiler from using optimal access patterns. For + // example, suppose a run of bit-fields occupies four bytes in a struct. If we + // split that into four 1-byte access units, then a sequence of assignments + // that doesn't touch all four bytes may have to be emitted with multiple + // 8-bit stores instead of a single 32-bit store. On the other hand, if we use + // very wide access units, we may find ourselves emitting accesses to + // bit-fields we didn't really need to touch, just because LLVM was unable to + // clean up after us. + + // It is desirable to have access units be aligned powers of 2 no larger than + // a register. (On non-strict alignment ISAs, the alignment requirement can be + // dropped.) A three byte access unit will be accessed using 2-byte and 1-byte + // accesses and bit manipulation. If no bitfield straddles across the two + // separate accesses, it is better to have separate 2-byte and 1-byte access + // units, as then LLVM will not generate unnecessary memory accesses, or bit + // manipulation. Similarly, on a strict-alignment architecture, it is better + // to keep access-units naturally aligned, to avoid similar bit + // manipulation synthesizing larger unaligned accesses. + + // Bitfields that share parts of a single byte are, of necessity, placed in + // the same access unit. That unit will encompass a consecutive run where + // adjacent bitfields share parts of a byte. (The first bitfield of such an + // access unit will start at the beginning of a byte.) + + // We then try and accumulate adjacent access units when the combined unit is + // naturally sized, no larger than a register, and (on a strict alignment + // ISA), naturally aligned. Note that this requires lookahead to one or more + // subsequent access units. For instance, consider a 2-byte access-unit + // followed by 2 1-byte units. We can merge that into a 4-byte access-unit, + // but we would not want to merge a 2-byte followed by a single 1-byte (and no + // available tail padding). We keep track of the best access unit seen so far, + // and use that when we determine we cannot accumulate any more. Then we start + // again at the bitfield following that best one. + + // The accumulation is also prevented when: + // *) it would cross a character-aigned zero-width bitfield, or + // *) fine-grained bitfield access option is in effect. + + CharUnits RegSize = + bitsToCharUnits(Context.getTargetInfo().getRegisterWidth()); + unsigned CharBits = Context.getCharWidth(); + + // Limit of useable tail padding at end of the record. Computed lazily and + // cached here. + CharUnits ScissorOffset = CharUnits::Zero(); + + // Data about the start of the span we're accumulating to create an access + // unit from. Begin is the first bitfield of the span. If Begin is FieldEnd, + // we've not got a current span. The span starts at the BeginOffset character + // boundary. BitSizeSinceBegin is the size (in bits) of the span -- this might + // include padding when we've advanced to a subsequent bitfield run. + RecordDecl::field_iterator Begin = FieldEnd; + CharUnits BeginOffset; + uint64_t BitSizeSinceBegin; + + // The (non-inclusive) end of the largest acceptable access unit we've found + // since Begin. If this is Begin, we're gathering the initial set of bitfields + // of a new span. BestEndOffset is the end of that acceptable access unit -- + // it might extend beyond the last character of the bitfield run, using + // available padding characters. + RecordDecl::field_iterator BestEnd = Begin; + CharUnits BestEndOffset; + bool BestClipped; // Whether the representation must be in a byte array. - // The start field is better as a single field run. - bool StartFieldAsSingleRun = false; for (;;) { - // Check to see if we need to start a new run. - if (Run == FieldEnd) { - // If we're out of fields, return. - if (Field == FieldEnd) + // AtAlignedBoundary is true iff Field is the (potential) start of a new + // span (or the end of the bitfields). When true, LimitOffset is the + // character offset of that span and Barrier indicates whether the new + // span cannot be merged into the current one. + bool AtAlignedBoundary = false; + bool Barrier = false; + + if (Field != FieldEnd && Field->isBitField()) { + uint64_t BitOffset = getFieldBitOffset(*Field); + if (Begin == FieldEnd) { + // Beginning a new span. + Begin = Field; + BestEnd = Begin; + + assert((BitOffset % CharBits) == 0 && "Not at start of char"); + BeginOffset = bitsToCharUnits(BitOffset); + BitSizeSinceBegin = 0; + } else if ((BitOffset % CharBits) != 0) { + // Bitfield occupies the same character as previous bitfield, it must be + // part of the same span. This can include zero-length bitfields, should + // the target not align them to character boundaries. Such non-alignment + // is at variance with the standards, which require zero-length + // bitfields be a barrier between access units. But of course we can't + // achieve that in the middle of a character. + assert(BitOffset == Context.toBits(BeginOffset) + BitSizeSinceBegin && + "Concatenating non-contiguous bitfields"); + } else { + // Bitfield potentially begins a new span. This includes zero-length + // bitfields on non-aligning targets that lie at character boundaries + // (those are barriers to merging). + if (Field->isZeroLengthBitField(Context)) + Barrier = true; + AtAlignedBoundary = true; + } + } else { + // We've reached the end of the bitfield run. Either we're done, or this + // is a barrier for the current span. + if (Begin == FieldEnd) break; - // Any non-zero-length bitfield can start a new run. - if (!Field->isZeroLengthBitField(Context)) { - Run = Field; - StartBitOffset = getFieldBitOffset(*Field); - Tail = StartBitOffset + Field->getBitWidthValue(Context); - StartFieldAsSingleRun = IsBetterAsSingleFieldRun(Tail - StartBitOffset, - StartBitOffset); + + Barrier = true; + AtAlignedBoundary = true; + } + + // InstallBest indicates whether we should create an access unit for the + // current best span: fields [Begin, BestEnd) occupying characters + // [BeginOffset, BestEndOffset). + bool InstallBest = false; + if (AtAlignedBoundary) { + // Field is the start of a new span or the end of the bitfields. The + // just-seen span now extends to BitSizeSinceBegin. + + // Determine if we can accumulate that just-seen span into the current + // accumulation. + CharUnits AccessSize = bitsToCharUnits(BitSizeSinceBegin + CharBits - 1); + if (BestEnd == Begin) { + // This is the initial run at the start of a new span. By definition, + // this is the best seen so far. + BestEnd = Field; + BestEndOffset = BeginOffset + AccessSize; + // Assume clipped until proven not below. + BestClipped = true; + if (!BitSizeSinceBegin) + // A zero-sized initial span -- this will install nothing and reset + // for another. + InstallBest = true; + } else if (AccessSize > RegSize) + // Accumulating the just-seen span would create a multi-register access + // unit, which would increase register pressure. + InstallBest = true; + + if (!InstallBest) { + // Determine if accumulating the just-seen span will create an expensive + // access unit or not. + llvm::Type *Type = getIntNType(Context.toBits(AccessSize)); + if (!Context.getTargetInfo().hasCheapUnalignedBitFieldAccess()) { + // Unaligned accesses are expensive. Only accumulate if the new unit + // is naturally aligned. Otherwise install the best we have, which is + // either the initial access unit (can't do better), or a naturally + // aligned accumulation (since we would have already installed it if + // it wasn't naturally aligned). + CharUnits Align = getAlignment(Type); + if (Align > Layout.getAlignment()) + // The alignment required is greater than the containing structure + // itself. + InstallBest = true; + else if (!BeginOffset.isMultipleOf(Align)) + // The access unit is not at a naturally aligned offset within the + // structure. + InstallBest = true; + + if (InstallBest && BestEnd == Field) + // We're installing the first span, whose clipping was presumed + // above. Compute it correctly. + if (getSize(Type) == AccessSize) + BestClipped = false; + } + + if (!InstallBest) { + // Find the next used storage offset to determine what the limit of + // the current span is. That's either the offset of the next field + // with storage (which might be Field itself) or the end of the + // non-reusable tail padding. + CharUnits LimitOffset; + for (auto Probe = Field; Probe != FieldEnd; ++Probe) + if (!isEmptyFieldForLayout(Context, *Probe)) { + // A member with storage sets the limit. + assert((getFieldBitOffset(*Probe) % CharBits) == 0 && + "Next storage is not byte-aligned"); + LimitOffset = bitsToCharUnits(getFieldBitOffset(*Probe)); + goto FoundLimit; + } + // We reached the end of the fields, determine the bounds of useable + // tail padding. As this can be complex for C++, we cache the result. + if (ScissorOffset.isZero()) { + ScissorOffset = calculateTailClippingOffset(isNonVirtualBaseType); + assert(!ScissorOffset.isZero() && "Tail clipping at zero"); + } + + LimitOffset = ScissorOffset; + FoundLimit:; + + CharUnits TypeSize = getSize(Type); + if (BeginOffset + TypeSize <= LimitOffset) { + // There is space before LimitOffset to create a naturally-sized + // access unit. + BestEndOffset = BeginOffset + TypeSize; + BestEnd = Field; + BestClipped = false; + } + + if (Barrier) + // The next field is a barrier that we cannot merge across. + InstallBest = true; + else if (Types.getCodeGenOpts().FineGrainedBitfieldAccesses) + // Fine-grained access, so no merging of spans. + InstallBest = true; + else + // Otherwise, we're not installing. Update the bit size + // of the current span to go all the way to LimitOffset, which is + // the (aligned) offset of next bitfield to consider. + BitSizeSinceBegin = Context.toBits(LimitOffset - BeginOffset); + } } - ++Field; - continue; } - // If the start field of a new run is better as a single run, or - // if current field (or consecutive fields) is better as a single run, or - // if current field has zero width bitfield and either - // UseZeroLengthBitfieldAlignment or UseBitFieldTypeAlignment is set to - // true, or - // if the offset of current field is inconsistent with the offset of - // previous field plus its offset, - // skip the block below and go ahead to emit the storage. - // Otherwise, try to add bitfields to the run. - if (!StartFieldAsSingleRun && Field != FieldEnd && - !IsBetterAsSingleFieldRun(Tail - StartBitOffset, StartBitOffset) && - (!Field->isZeroLengthBitField(Context) || - (!Context.getTargetInfo().useZeroLengthBitfieldAlignment() && - !Context.getTargetInfo().useBitFieldTypeAlignment())) && - Tail == getFieldBitOffset(*Field)) { - Tail += Field->getBitWidthValue(Context); + if (InstallBest) { + assert((Field == FieldEnd || !Field->isBitField() || + (getFieldBitOffset(*Field) % CharBits) == 0) && + "Installing but not at an aligned bitfield or limit"); + CharUnits AccessSize = BestEndOffset - BeginOffset; + if (!AccessSize.isZero()) { + // Add the storage member for the access unit to the record. The + // bitfields get the offset of their storage but come afterward and + // remain there after a stable sort. + llvm::Type *Type; + if (BestClipped) { + assert(getSize(getIntNType(Context.toBits(AccessSize))) > + AccessSize && + "Clipped access need not be clipped"); + Type = getByteArrayType(AccessSize); + } else { + Type = getIntNType(Context.toBits(AccessSize)); + assert(getSize(Type) == AccessSize && + "Unclipped access must be clipped"); + } + Members.push_back(StorageInfo(BeginOffset, Type)); + for (; Begin != BestEnd; ++Begin) + if (!Begin->isZeroLengthBitField(Context)) + Members.push_back( + MemberInfo(BeginOffset, MemberInfo::Field, nullptr, *Begin)); + } + // Reset to start a new span. + Field = BestEnd; + Begin = FieldEnd; + } else { + assert(Field != FieldEnd && Field->isBitField() && + "Accumulating past end of bitfields"); + assert(!Barrier && "Accumulating across barrier"); + // Accumulate this bitfield into the current (potential) span. + BitSizeSinceBegin += Field->getBitWidthValue(Context); ++Field; - continue; } - - // We've hit a break-point in the run and need to emit a storage field. - llvm::Type *Type = getIntNType(Tail - StartBitOffset); - // Add the storage member to the record and set the bitfield info for all of - // the bitfields in the run. Bitfields get the offset of their storage but - // come afterward and remain there after a stable sort. - Members.push_back(StorageInfo(bitsToCharUnits(StartBitOffset), Type)); - for (; Run != Field; ++Run) - Members.push_back(MemberInfo(bitsToCharUnits(StartBitOffset), - MemberInfo::Field, nullptr, *Run)); - Run = FieldEnd; - StartFieldAsSingleRun = false; } + + return Field; } void CGRecordLowering::accumulateBases() { @@ -532,7 +732,7 @@ void CGRecordLowering::accumulateBases() { // Bases can be zero-sized even if not technically empty if they // contain only a trailing array member. const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); - if (!BaseDecl->isEmpty() && + if (!isEmptyRecordForLayout(Context, Base.getType()) && !Context.getASTRecordLayout(BaseDecl).getNonVirtualSize().isZero()) Members.push_back(MemberInfo(Layout.getBaseClassOffset(BaseDecl), MemberInfo::Base, getStorageType(BaseDecl), BaseDecl)); @@ -667,16 +867,20 @@ void CGRecordLowering::accumulateVPtrs() { llvm::PointerType::getUnqual(Types.getLLVMContext()))); } -void CGRecordLowering::accumulateVBases() { +CharUnits +CGRecordLowering::calculateTailClippingOffset(bool isNonVirtualBaseType) const { + if (!RD) + return Layout.getDataSize(); + CharUnits ScissorOffset = Layout.getNonVirtualSize(); // In the itanium ABI, it's possible to place a vbase at a dsize that is // smaller than the nvsize. Here we check to see if such a base is placed // before the nvsize and set the scissor offset to that, instead of the // nvsize. - if (isOverlappingVBaseABI()) + if (!isNonVirtualBaseType && isOverlappingVBaseABI()) for (const auto &Base : RD->vbases()) { const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); - if (BaseDecl->isEmpty()) + if (isEmptyRecordForLayout(Context, Base.getType())) continue; // If the vbase is a primary virtual base of some base, then it doesn't // get its own storage location but instead lives inside of that base. @@ -685,11 +889,14 @@ void CGRecordLowering::accumulateVBases() { ScissorOffset = std::min(ScissorOffset, Layout.getVBaseClassOffset(BaseDecl)); } - Members.push_back(MemberInfo(ScissorOffset, MemberInfo::Scissor, nullptr, - RD)); + + return ScissorOffset; +} + +void CGRecordLowering::accumulateVBases() { for (const auto &Base : RD->vbases()) { const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); - if (BaseDecl->isEmpty()) + if (isEmptyRecordForLayout(Context, Base.getType())) continue; CharUnits Offset = Layout.getVBaseClassOffset(BaseDecl); // If the vbase is a primary virtual base of some base, then it doesn't @@ -711,7 +918,7 @@ void CGRecordLowering::accumulateVBases() { } bool CGRecordLowering::hasOwnStorage(const CXXRecordDecl *Decl, - const CXXRecordDecl *Query) { + const CXXRecordDecl *Query) const { const ASTRecordLayout &DeclLayout = Context.getASTRecordLayout(Decl); if (DeclLayout.isPrimaryBaseVirtual() && DeclLayout.getPrimaryBase() == Query) return false; @@ -740,32 +947,22 @@ void CGRecordLowering::calculateZeroInit() { } } -void CGRecordLowering::clipTailPadding() { - std::vector<MemberInfo>::iterator Prior = Members.begin(); - CharUnits Tail = getSize(Prior->Data); - for (std::vector<MemberInfo>::iterator Member = Prior + 1, - MemberEnd = Members.end(); - Member != MemberEnd; ++Member) { - // Only members with data and the scissor can cut into tail padding. - if (!Member->Data && Member->Kind != MemberInfo::Scissor) +// Verify accumulateBitfields computed the correct storage representations. +void CGRecordLowering::checkBitfieldClipping(bool IsNonVirtualBaseType) const { +#ifndef NDEBUG + auto ScissorOffset = calculateTailClippingOffset(IsNonVirtualBaseType); + auto Tail = CharUnits::Zero(); + for (const auto &M : Members) { + // Only members with data could possibly overlap. + if (!M.Data) continue; - if (Member->Offset < Tail) { - assert(Prior->Kind == MemberInfo::Field && - "Only storage fields have tail padding!"); - if (!Prior->FD || Prior->FD->isBitField()) - Prior->Data = getByteArrayType(bitsToCharUnits(llvm::alignTo( - cast<llvm::IntegerType>(Prior->Data)->getIntegerBitWidth(), 8))); - else { - assert(Prior->FD->hasAttr<NoUniqueAddressAttr>() && - "should not have reused this field's tail padding"); - Prior->Data = getByteArrayType( - Context.getTypeInfoDataSizeInChars(Prior->FD->getType()).Width); - } - } - if (Member->Data) - Prior = Member; - Tail = Prior->Offset + getSize(Prior->Data); + + assert(M.Offset >= Tail && "Bitfield access unit is not clipped"); + Tail = M.Offset + getSize(M.Data); + assert((Tail <= ScissorOffset || M.Offset >= ScissorOffset) && + "Bitfield straddles scissor offset"); } +#endif } void CGRecordLowering::determinePacked(bool NVBaseType) { @@ -965,7 +1162,7 @@ CodeGenTypes::ComputeRecordLayout(const RecordDecl *D, llvm::StructType *Ty) { const FieldDecl *FD = *it; // Ignore zero-sized fields. - if (FD->isZeroSize(getContext())) + if (isEmptyFieldForLayout(getContext(), FD)) continue; // For non-bit-fields, just check that the LLVM struct offset matches the diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGStmt.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGStmt.cpp index beff0ad9da27..2f466602d2f6 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGStmt.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGStmt.cpp @@ -43,6 +43,10 @@ using namespace CodeGen; // Statement Emission //===----------------------------------------------------------------------===// +namespace llvm { +extern cl::opt<bool> EnableSingleByteCoverage; +} // namespace llvm + void CodeGenFunction::EmitStopPoint(const Stmt *S) { if (CGDebugInfo *DI = getDebugInfo()) { SourceLocation Loc; @@ -218,6 +222,12 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) { case Stmt::OMPUnrollDirectiveClass: EmitOMPUnrollDirective(cast<OMPUnrollDirective>(*S)); break; + case Stmt::OMPReverseDirectiveClass: + EmitOMPReverseDirective(cast<OMPReverseDirective>(*S)); + break; + case Stmt::OMPInterchangeDirectiveClass: + EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S)); + break; case Stmt::OMPForDirectiveClass: EmitOMPForDirective(cast<OMPForDirective>(*S)); break; @@ -410,7 +420,8 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) { CGM.ErrorUnsupported(S, "OpenMP dispatch directive"); break; case Stmt::OMPScopeDirectiveClass: - llvm_unreachable("scope not supported with FE outlining"); + CGM.ErrorUnsupported(S, "scope with FE outlining"); + break; case Stmt::OMPMaskedDirectiveClass: EmitOMPMaskedDirective(cast<OMPMaskedDirective>(*S)); break; @@ -435,6 +446,12 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) { case Stmt::OMPParallelMaskedDirectiveClass: EmitOMPParallelMaskedDirective(cast<OMPParallelMaskedDirective>(*S)); break; + case Stmt::OpenACCComputeConstructClass: + EmitOpenACCComputeConstruct(cast<OpenACCComputeConstruct>(*S)); + break; + case Stmt::OpenACCLoopConstructClass: + EmitOpenACCLoopConstruct(cast<OpenACCLoopConstruct>(*S)); + break; } } @@ -721,11 +738,19 @@ void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) { case attr::AlwaysInline: alwaysinline = true; break; - case attr::MustTail: + case attr::MustTail: { const Stmt *Sub = S.getSubStmt(); const ReturnStmt *R = cast<ReturnStmt>(Sub); musttail = cast<CallExpr>(R->getRetValue()->IgnoreParens()); - break; + } break; + case attr::CXXAssume: { + const Expr *Assumption = cast<CXXAssumeAttr>(A)->getAssumption(); + if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() && + !Assumption->HasSideEffects(getContext())) { + llvm::Value *AssumptionVal = EvaluateExprAsBool(Assumption); + Builder.CreateAssumption(AssumptionVal); + } + } break; } } SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge); @@ -853,7 +878,10 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // Emit the 'then' code. EmitBlock(ThenBlock); - incrementProfileCounter(&S); + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getThen()); + else + incrementProfileCounter(&S); { RunCleanupsScope ThenScope(*this); EmitStmt(S.getThen()); @@ -867,6 +895,9 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { auto NL = ApplyDebugLocation::CreateEmpty(*this); EmitBlock(ElseBlock); } + // When single byte coverage mode is enabled, add a counter to else block. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(Else); { RunCleanupsScope ElseScope(*this); EmitStmt(Else); @@ -880,6 +911,74 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // Emit the continuation block for code after the if. EmitBlock(ContBlock, true); + + // When single byte coverage mode is enabled, add a counter to continuation + // block. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(&S); +} + +bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression, + bool HasEmptyBody) { + if (CGM.getCodeGenOpts().getFiniteLoops() == + CodeGenOptions::FiniteLoopsKind::Never) + return false; + + // Now apply rules for plain C (see 6.8.5.6 in C11). + // Loops with constant conditions do not have to make progress in any C + // version. + // As an extension, we consisider loops whose constant expression + // can be constant-folded. + Expr::EvalResult Result; + bool CondIsConstInt = + !ControllingExpression || + (ControllingExpression->EvaluateAsInt(Result, getContext()) && + Result.Val.isInt()); + + bool CondIsTrue = CondIsConstInt && (!ControllingExpression || + Result.Val.getInt().getBoolValue()); + + // Loops with non-constant conditions must make progress in C11 and later. + if (getLangOpts().C11 && !CondIsConstInt) + return true; + + // [C++26][intro.progress] (DR) + // The implementation may assume that any thread will eventually do one of the + // following: + // [...] + // - continue execution of a trivial infinite loop ([stmt.iter.general]). + if (CGM.getCodeGenOpts().getFiniteLoops() == + CodeGenOptions::FiniteLoopsKind::Always || + getLangOpts().CPlusPlus11) { + if (HasEmptyBody && CondIsTrue) { + CurFn->removeFnAttr(llvm::Attribute::MustProgress); + return false; + } + return true; + } + return false; +} + +// [C++26][stmt.iter.general] (DR) +// A trivially empty iteration statement is an iteration statement matching one +// of the following forms: +// - while ( expression ) ; +// - while ( expression ) { } +// - do ; while ( expression ) ; +// - do { } while ( expression ) ; +// - for ( init-statement expression(opt); ) ; +// - for ( init-statement expression(opt); ) { } +template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) { + if constexpr (std::is_same_v<LoopStmt, ForStmt>) { + if (S.getInc()) + return false; + } + const Stmt *Body = S.getBody(); + if (!Body || isa<NullStmt>(Body)) + return true; + if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Body)) + return Compound->body_empty(); + return false; } void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, @@ -889,6 +988,10 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); EmitBlock(LoopHeader.getBlock()); + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.push_back(emitConvergenceLoopToken( + LoopHeader.getBlock(), ConvergenceTokenStack.back())); + // Create an exit block for when the condition fails, which will // also become the break target. JumpDest LoopExit = getJumpDestInCurrentScope("while.end"); @@ -916,13 +1019,16 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, // while(1) is common, avoid extra exit blocks. Be sure // to correctly handle break/continue though. llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal); - bool CondIsConstInt = C != nullptr; - bool EmitBoolCondBranch = !CondIsConstInt || !C->isOne(); + bool EmitBoolCondBranch = !C || !C->isOne(); const SourceRange &R = S.getSourceRange(); LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), CGM.getCodeGenOpts(), WhileAttrs, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd()), - checkIfLoopMustProgress(CondIsConstInt)); + checkIfLoopMustProgress(S.getCond(), hasEmptyLoopBody(S))); + + // When single byte coverage mode is enabled, add a counter to loop condition. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getCond()); // As long as the condition is true, go to the loop body. llvm::BasicBlock *LoopBody = createBasicBlock("while.body"); @@ -956,7 +1062,11 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, { RunCleanupsScope BodyScope(*this); EmitBlock(LoopBody); - incrementProfileCounter(&S); + // When single byte coverage mode is enabled, add a counter to the body. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getBody()); + else + incrementProfileCounter(&S); EmitStmt(S.getBody()); } @@ -978,6 +1088,14 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, // a branch, try to erase it. if (!EmitBoolCondBranch) SimplifyForwardingBlocks(LoopHeader.getBlock()); + + // When single byte coverage mode is enabled, add a counter to continuation + // block. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(&S); + + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.pop_back(); } void CodeGenFunction::EmitDoStmt(const DoStmt &S, @@ -993,13 +1111,24 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); - EmitBlockWithFallThrough(LoopBody, &S); + if (llvm::EnableSingleByteCoverage) + EmitBlockWithFallThrough(LoopBody, S.getBody()); + else + EmitBlockWithFallThrough(LoopBody, &S); + + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.push_back( + emitConvergenceLoopToken(LoopBody, ConvergenceTokenStack.back())); + { RunCleanupsScope BodyScope(*this); EmitStmt(S.getBody()); } EmitBlock(LoopCond.getBlock()); + // When single byte coverage mode is enabled, add a counter to loop condition. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getCond()); // C99 6.8.5.2: "The evaluation of the controlling expression takes place // after each execution of the loop body." @@ -1014,14 +1143,13 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // "do {} while (0)" is common in macros, avoid extra blocks. Be sure // to correctly handle break/continue though. llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal); - bool CondIsConstInt = C; bool EmitBoolCondBranch = !C || !C->isZero(); const SourceRange &R = S.getSourceRange(); LoopStack.push(LoopBody, CGM.getContext(), CGM.getCodeGenOpts(), DoAttrs, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd()), - checkIfLoopMustProgress(CondIsConstInt)); + checkIfLoopMustProgress(S.getCond(), hasEmptyLoopBody(S))); // As long as the condition is true, iterate the loop. if (EmitBoolCondBranch) { @@ -1040,6 +1168,14 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // emitting a branch, try to erase it. if (!EmitBoolCondBranch) SimplifyForwardingBlocks(LoopCond.getBlock()); + + // When single byte coverage mode is enabled, add a counter to continuation + // block. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(&S); + + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.pop_back(); } void CodeGenFunction::EmitForStmt(const ForStmt &S, @@ -1059,15 +1195,15 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, llvm::BasicBlock *CondBlock = CondDest.getBlock(); EmitBlock(CondBlock); - Expr::EvalResult Result; - bool CondIsConstInt = - !S.getCond() || S.getCond()->EvaluateAsInt(Result, getContext()); + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.push_back( + emitConvergenceLoopToken(CondBlock, ConvergenceTokenStack.back())); const SourceRange &R = S.getSourceRange(); LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd()), - checkIfLoopMustProgress(CondIsConstInt)); + checkIfLoopMustProgress(S.getCond(), hasEmptyLoopBody(S))); // Create a cleanup scope for the condition variable cleanups. LexicalScope ConditionScope(*this, S.getSourceRange()); @@ -1098,6 +1234,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, BreakContinueStack.back().ContinueBlock = Continue; } + // When single byte coverage mode is enabled, add a counter to loop + // condition. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getCond()); + llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. @@ -1128,8 +1269,12 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // Treat it as a non-zero constant. Don't even create a new block for the // body, just fall into it. } - incrementProfileCounter(&S); + // When single byte coverage mode is enabled, add a counter to the body. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getBody()); + else + incrementProfileCounter(&S); { // Create a separate cleanup scope for the body, in case it is not // a compound statement. @@ -1141,6 +1286,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, if (S.getInc()) { EmitBlock(Continue.getBlock()); EmitStmt(S.getInc()); + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getInc()); } BreakContinueStack.pop_back(); @@ -1156,6 +1303,14 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // Emit the fall-through block. EmitBlock(LoopExit.getBlock(), true); + + // When single byte coverage mode is enabled, add a counter to continuation + // block. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(&S); + + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.pop_back(); } void @@ -1178,6 +1333,10 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, llvm::BasicBlock *CondBlock = createBasicBlock("for.cond"); EmitBlock(CondBlock); + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.push_back( + emitConvergenceLoopToken(CondBlock, ConvergenceTokenStack.back())); + const SourceRange &R = S.getSourceRange(); LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs, SourceLocToDebugLoc(R.getBegin()), @@ -1208,7 +1367,10 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, } EmitBlock(ForBody); - incrementProfileCounter(&S); + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(S.getBody()); + else + incrementProfileCounter(&S); // Create a block for the increment. In case of a 'continue', we jump there. JumpDest Continue = getJumpDestInCurrentScope("for.inc"); @@ -1238,6 +1400,14 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, // Emit the fall-through block. EmitBlock(LoopExit.getBlock(), true); + + // When single byte coverage mode is enabled, add a counter to continuation + // block. + if (llvm::EnableSingleByteCoverage) + incrementProfileCounter(&S); + + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.pop_back(); } void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) { @@ -1267,10 +1437,8 @@ struct SaveRetExprRAII { }; } // namespace -/// If we have 'return f(...);', where both caller and callee are SwiftAsync, -/// codegen it as 'tail call ...; ret void;'. -static void makeTailCallIfSwiftAsync(const CallExpr *CE, CGBuilderTy &Builder, - const CGFunctionInfo *CurFnInfo) { +/// Determine if the given call uses the swiftasync calling convention. +static bool isSwiftAsyncCallee(const CallExpr *CE) { auto calleeQualType = CE->getCallee()->getType(); const FunctionType *calleeType = nullptr; if (calleeQualType->isFunctionPointerType() || @@ -1285,18 +1453,12 @@ static void makeTailCallIfSwiftAsync(const CallExpr *CE, CGBuilderTy &Builder, // getMethodDecl() doesn't handle member pointers at the moment. calleeType = methodDecl->getType()->castAs<FunctionType>(); } else { - return; + return false; } } else { - return; - } - if (calleeType->getCallConv() == CallingConv::CC_SwiftAsync && - (CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync)) { - auto CI = cast<llvm::CallInst>(&Builder.GetInsertBlock()->back()); - CI->setTailCallKind(llvm::CallInst::TCK_MustTail); - Builder.CreateRetVoid(); - Builder.ClearInsertionPoint(); + return false; } + return calleeType->getCallConv() == CallingConv::CC_SwiftAsync; } /// EmitReturnStmt - Note that due to GCC extensions, this can have an operand @@ -1336,6 +1498,19 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { RunCleanupsScope cleanupScope(*this); if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(RV)) RV = EWC->getSubExpr(); + + // If we're in a swiftasynccall function, and the return expression is a + // call to a swiftasynccall function, mark the call as the musttail call. + std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail; + if (RV && CurFnInfo && + CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) { + if (auto CE = dyn_cast<CallExpr>(RV)) { + if (isSwiftAsyncCallee(CE)) { + SaveMustTail.emplace(MustTailCall, CE); + } + } + } + // FIXME: Clean this up by using an LValue for ReturnTemp, // EmitStoreThroughLValue, and EmitAnyExpr. // Check if the NRVO candidate was not globalized in OpenMP mode. @@ -1358,8 +1533,6 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { // for side effects. if (RV) { EmitAnyExpr(RV); - if (auto *CE = dyn_cast<CallExpr>(RV)) - makeTailCallIfSwiftAsync(CE, Builder, CurFnInfo); } } else if (!RV) { // Do nothing (return value is left uninitialized) @@ -1370,9 +1543,15 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { Builder.CreateStore(Result.getScalarVal(), ReturnValue); } else { switch (getEvaluationKind(RV->getType())) { - case TEK_Scalar: - Builder.CreateStore(EmitScalarExpr(RV), ReturnValue); + case TEK_Scalar: { + llvm::Value *Ret = EmitScalarExpr(RV); + if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) + EmitStoreOfScalar(Ret, MakeAddrLValue(ReturnValue, RV->getType()), + /*isInit*/ true); + else + Builder.CreateStore(Ret, ReturnValue); break; + } case TEK_Complex: EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()), /*isInit*/ true); @@ -2209,15 +2388,14 @@ std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue( getTargetHooks().isScalarizableAsmOperand(*this, Ty)) { Ty = llvm::IntegerType::get(getLLVMContext(), Size); - return { - Builder.CreateLoad(InputValue.getAddress(*this).withElementType(Ty)), - nullptr}; + return {Builder.CreateLoad(InputValue.getAddress().withElementType(Ty)), + nullptr}; } } - Address Addr = InputValue.getAddress(*this); + Address Addr = InputValue.getAddress(); ConstraintStr += '*'; - return {Addr.getPointer(), Addr.getElementType()}; + return {InputValue.getPointer(*this), Addr.getElementType()}; } std::pair<llvm::Value *, llvm::Type *> @@ -2411,7 +2589,7 @@ EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S, // ResultTypeRequiresCast.size() elements of RegResults. if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) { unsigned Size = CGF.getContext().getTypeSize(ResultRegQualTys[i]); - Address A = Dest.getAddress(CGF).withElementType(ResultRegTypes[i]); + Address A = Dest.getAddress().withElementType(ResultRegTypes[i]); if (CGF.getTargetHooks().isScalarizableAsmOperand(CGF, TruncTy)) { Builder.CreateStore(Tmp, A); continue; @@ -2613,7 +2791,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { std::max((uint64_t)LargestVectorWidth, VT->getPrimitiveSizeInBits().getKnownMinValue()); } else { - Address DestAddr = Dest.getAddress(*this); + Address DestAddr = Dest.getAddress(); // Matrix types in memory are represented by arrays, but accessed through // vector pointers, with the alignment specified on the access operation. // For inline assembly, update pointer arguments to use vector pointers. @@ -2624,7 +2802,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { ArgTypes.push_back(DestAddr.getType()); ArgElemTypes.push_back(DestAddr.getElementType()); - Args.push_back(DestAddr.getPointer()); + Args.push_back(DestAddr.emitRawPointer(*this)); Constraints += "=*"; Constraints += OutputConstraint; ReadOnly = ReadNone = false; @@ -2961,7 +3139,7 @@ CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) { Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) { LValue CapStruct = InitCapturedStruct(S); - return CapStruct.getAddress(*this); + return CapStruct.getAddress(); } /// Creates the outlined function for a CapturedStmt. @@ -2999,8 +3177,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr)); // Initialize variable-length arrays. - LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(), - Ctx.getTagDeclType(RD)); + LValue Base = MakeNaturalAlignRawAddrLValue( + CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD)); for (auto *FD : RD->fields()) { if (FD->hasCapturedVLAType()) { auto *ExprArg = @@ -3024,3 +3202,67 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { return F; } + +namespace { +// Returns the first convergence entry/loop/anchor instruction found in |BB|. +// std::nullptr otherwise. +llvm::IntrinsicInst *getConvergenceToken(llvm::BasicBlock *BB) { + for (auto &I : *BB) { + auto *II = dyn_cast<llvm::IntrinsicInst>(&I); + if (II && llvm::isConvergenceControlIntrinsic(II->getIntrinsicID())) + return II; + } + return nullptr; +} + +} // namespace + +llvm::CallBase * +CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input, + llvm::Value *ParentToken) { + llvm::Value *bundleArgs[] = {ParentToken}; + llvm::OperandBundleDef OB("convergencectrl", bundleArgs); + auto Output = llvm::CallBase::addOperandBundle( + Input, llvm::LLVMContext::OB_convergencectrl, OB, Input); + Input->replaceAllUsesWith(Output); + Input->eraseFromParent(); + return Output; +} + +llvm::IntrinsicInst * +CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB, + llvm::Value *ParentToken) { + CGBuilderTy::InsertPoint IP = Builder.saveIP(); + if (BB->empty()) + Builder.SetInsertPoint(BB); + else + Builder.SetInsertPoint(BB->getFirstInsertionPt()); + + llvm::CallBase *CB = Builder.CreateIntrinsic( + llvm::Intrinsic::experimental_convergence_loop, {}, {}); + Builder.restoreIP(IP); + + llvm::CallBase *I = addConvergenceControlToken(CB, ParentToken); + return cast<llvm::IntrinsicInst>(I); +} + +llvm::IntrinsicInst * +CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) { + llvm::BasicBlock *BB = &F->getEntryBlock(); + llvm::IntrinsicInst *Token = getConvergenceToken(BB); + if (Token) + return Token; + + // Adding a convergence token requires the function to be marked as + // convergent. + F->setConvergent(); + + CGBuilderTy::InsertPoint IP = Builder.saveIP(); + Builder.SetInsertPoint(&BB->front()); + llvm::CallBase *I = Builder.CreateIntrinsic( + llvm::Intrinsic::experimental_convergence_entry, {}, {}); + assert(isa<llvm::IntrinsicInst>(I)); + Builder.restoreIP(IP); + + return cast<llvm::IntrinsicInst>(I); +} diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp index e362c9da51fe..adf74ea16c89 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -24,6 +24,7 @@ #include "clang/AST/StmtVisitor.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/PrettyStackTrace.h" +#include "clang/Basic/SourceManager.h" #include "llvm/ADT/SmallSet.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/Frontend/OpenMP/OMPConstants.h" @@ -34,11 +35,14 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Debug.h" #include <optional> using namespace clang; using namespace CodeGen; using namespace llvm::omp; +#define TTL_CODEGEN_TYPE "target-teams-loop-codegen" + static const VarDecl *getBaseDecl(const Expr *Ref); namespace { @@ -68,7 +72,7 @@ class OMPLexicalScope : public CodeGenFunction::LexicalScope { static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || - (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && + (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) && cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); } @@ -96,7 +100,7 @@ public: isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); - InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); + InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress()); } } (void)InlinedShareds.Privatize(); @@ -138,7 +142,7 @@ public: /// of used expression from loop statement. class OMPLoopScope : public CodeGenFunction::RunCleanupsScope { void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) { - const DeclStmt *PreInits; + const Stmt *PreInits; CodeGenFunction::OMPMapVars PreCondVars; if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) { llvm::DenseSet<const VarDecl *> EmittedAsPrivate; @@ -178,17 +182,39 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope { } return false; }); - PreInits = cast_or_null<DeclStmt>(LD->getPreInits()); + PreInits = LD->getPreInits(); } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) { - PreInits = cast_or_null<DeclStmt>(Tile->getPreInits()); + PreInits = Tile->getPreInits(); } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) { - PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits()); + PreInits = Unroll->getPreInits(); + } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) { + PreInits = Reverse->getPreInits(); + } else if (const auto *Interchange = + dyn_cast<OMPInterchangeDirective>(&S)) { + PreInits = Interchange->getPreInits(); } else { llvm_unreachable("Unknown loop-based directive kind."); } if (PreInits) { - for (const auto *I : PreInits->decls()) - CGF.EmitVarDecl(cast<VarDecl>(*I)); + // CompoundStmts and DeclStmts are used as lists of PreInit statements and + // declarations. Since declarations must be visible in the the following + // that they initialize, unpack the CompoundStmt they are nested in. + SmallVector<const Stmt *> PreInitStmts; + if (auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits)) + llvm::append_range(PreInitStmts, PreInitCompound->body()); + else + PreInitStmts.push_back(PreInits); + + for (const Stmt *S : PreInitStmts) { + // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted + // here. + if (auto *PreInitDecl = dyn_cast<DeclStmt>(S)) { + for (Decl *I : PreInitDecl->decls()) + CGF.EmitVarDecl(cast<VarDecl>(*I)); + continue; + } + CGF.EmitStmt(S); + } } PreCondVars.restore(CGF); } @@ -206,7 +232,7 @@ class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope { static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || - (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && + (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) && cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); } @@ -272,7 +298,7 @@ public: InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); - InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); + InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress()); } } CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt()); @@ -294,7 +320,7 @@ LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) { bool IsCaptured = LambdaCaptureFields.lookup(OrigVD) || (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) || - (CurCodeDecl && isa<BlockDecl>(CurCodeDecl)); + (isa_and_nonnull<BlockDecl>(CurCodeDecl)); DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured, OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc()); return EmitLValue(&DRE); @@ -350,7 +376,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); llvm::Value *SrcAddrVal = EmitScalarConversion( - DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), + DstAddr.emitRawPointer(*this), + Ctx.getPointerType(Ctx.getUIntPtrType()), Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); LValue SrcLV = MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); @@ -364,7 +391,7 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( CapturedVars.push_back(CV); } else { assert(CurCap->capturesVariable() && "Expected capture by reference."); - CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer()); + CapturedVars.push_back(EmitLValue(*I).getAddress().emitRawPointer(*this)); } } } @@ -375,10 +402,11 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, ASTContext &Ctx = CGF.getContext(); llvm::Value *CastedPtr = CGF.EmitScalarConversion( - AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), + AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(), Ctx.getPointerType(DstType), Loc); + // FIXME: should the pointee type (DstType) be passed? Address TmpAddr = - CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); + CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(); return TmpAddr; } @@ -571,7 +599,7 @@ static llvm::Function *emitOutlinedFunctionPrologue( } else if (I->capturesVariable()) { const VarDecl *Var = I->getCapturedVar(); QualType VarTy = Var->getType(); - Address ArgAddr = ArgLVal.getAddress(CGF); + Address ArgAddr = ArgLVal.getAddress(); if (ArgLVal.getType()->isLValueReferenceType()) { ArgAddr = CGF.EmitLoadOfReference(ArgLVal); } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { @@ -592,12 +620,12 @@ static llvm::Function *emitOutlinedFunctionPrologue( ? castValueFromUintptr( CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal) - : ArgLVal.getAddress(CGF)}}); + : ArgLVal.getAddress()}}); } else { // If 'this' is captured, load it into CXXThisValue. assert(I->capturesThis()); CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); - LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); + LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}}); } ++Cnt; ++I; @@ -667,7 +695,7 @@ CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, I->second.first ? I->second.first->getType() : Arg->getType(), AlignmentSource::Decl); if (LV.getType()->isAnyComplexType()) - LV.setAddress(LV.getAddress(WrapperCGF).withElementType(PI->getType())); + LV.setAddress(LV.getAddress().withElementType(PI->getType())); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc()); } else { auto EI = VLASizes.find(Arg); @@ -702,8 +730,8 @@ void CodeGenFunction::EmitOMPAggregateAssign( llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); SrcAddr = SrcAddr.withElementType(DestAddr.getElementType()); - llvm::Value *SrcBegin = SrcAddr.getPointer(); - llvm::Value *DestBegin = DestAddr.getPointer(); + llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this); + llvm::Value *DestBegin = DestAddr.emitRawPointer(*this); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -883,8 +911,7 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, EmitAggregateAssign(Dest, OriginalLVal, Type); } else { EmitOMPAggregateAssign( - Emission.getAllocatedAddress(), OriginalLVal.getAddress(*this), - Type, + Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type, [this, VDInit, Init](Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the // initialization. @@ -901,7 +928,7 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, IsRegistered = PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress()); } else { - Address OriginalAddr = OriginalLVal.getAddress(*this); + Address OriginalAddr = OriginalLVal.getAddress(); // Emit private VarDecl with copy init. // Remap temp VDInit variable to the address of the original // variable (for proper handling of captured global variables). @@ -990,7 +1017,7 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { "Copyin threadprivates should have been captured!"); DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); - MasterAddr = EmitLValue(&DRE).getAddress(*this); + MasterAddr = EmitLValue(&DRE).getAddress(); LocalDeclMap.erase(VD); } else { MasterAddr = @@ -1000,17 +1027,17 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { getContext().getDeclAlign(VD)); } // Get the address of the threadprivate variable. - Address PrivateAddr = EmitLValue(*IRef).getAddress(*this); + Address PrivateAddr = EmitLValue(*IRef).getAddress(); if (CopiedVars.size() == 1) { // At first check if current thread is a master thread. If it is, no // need to copy data. CopyBegin = createBasicBlock("copyin.not.master"); CopyEnd = createBasicBlock("copyin.not.master.end"); // TODO: Avoid ptrtoint conversion. - auto *MasterAddrInt = - Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy); - auto *PrivateAddrInt = - Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy); + auto *MasterAddrInt = Builder.CreatePtrToInt( + MasterAddr.emitRawPointer(*this), CGM.IntPtrTy); + auto *PrivateAddrInt = Builder.CreatePtrToInt( + PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy); Builder.CreateCondBr( Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin, CopyEnd); @@ -1069,7 +1096,7 @@ bool CodeGenFunction::EmitOMPLastprivateClauseInit( /*RefersToEnclosingVariableOrCapture=*/ CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); - PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress(*this)); + PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress()); // Check if the variable is also a firstprivate: in this case IInit is // not generated. Initialization of this variable will happen in codegen // for 'firstprivate' clause. @@ -1232,7 +1259,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit( RedCG.emitAggregateType(*this, Count); AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(), - RedCG.getSharedLValue(Count).getAddress(*this), + RedCG.getSharedLValue(Count).getAddress(), [&Emission](CodeGenFunction &CGF) { CGF.EmitAutoVarInit(Emission); return true; @@ -1249,26 +1276,24 @@ void CodeGenFunction::EmitOMPReductionClauseInit( const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); QualType Type = PrivateVD->getType(); - bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef); + bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef); if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) { // Store the address of the original variable associated with the LHS // implicit variable. - PrivateScope.addPrivate(LHSVD, - RedCG.getSharedLValue(Count).getAddress(*this)); + PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress()); PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD)); } else if ((isaOMPArraySectionExpr && Type->isScalarType()) || isa<ArraySubscriptExpr>(IRef)) { // Store the address of the original variable associated with the LHS // implicit variable. - PrivateScope.addPrivate(LHSVD, - RedCG.getSharedLValue(Count).getAddress(*this)); + PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress()); PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD).withElementType( ConvertTypeForMem(RHSVD->getType()))); } else { QualType Type = PrivateVD->getType(); bool IsArray = getContext().getAsArrayType(Type) != nullptr; - Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this); + Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(); // Store the address of the original variable associated with the LHS // implicit variable. if (IsArray) { @@ -1391,7 +1416,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit( case OMPD_end_declare_variant: case OMPD_unknown: default: - llvm_unreachable("Enexpected directive with task reductions."); + llvm_unreachable("Unexpected directive with task reductions."); } const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl()); @@ -1429,9 +1454,12 @@ void CodeGenFunction::EmitOMPReductionClauseFinal( *this, D.getBeginLoc(), isOpenMPWorksharingDirective(D.getDirectiveKind())); } + bool TeamsLoopCanBeParallel = false; + if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D)) + TeamsLoopCanBeParallel = TTLD->canBeParallelFor(); bool WithNowait = D.getSingleClause<OMPNowaitClause>() || isOpenMPParallelDirective(D.getDirectiveKind()) || - ReductionKind == OMPD_simd; + TeamsLoopCanBeParallel || ReductionKind == OMPD_simd; bool SimpleReduction = ReductionKind == OMPD_simd; // Emit nowait reduction if nowait clause is present or directive is a // parallel directive (it always has implicit barrier). @@ -1666,7 +1694,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Data = - CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy); llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)); std::string Suffix = getNameWithSeparators({"cache", ""}); llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix); @@ -1743,7 +1771,7 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - // The cleanup callback that finalizes all variabels at the given location, + // The cleanup callback that finalizes all variables at the given location, // thus calls destructors etc. auto FiniCB = [this](InsertPointTy IP) { OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP); @@ -2045,7 +2073,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { ->getParam(0) ->getType() .getNonReferenceType(); - Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); + RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()}); llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count"); @@ -2059,9 +2087,9 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { // variable and emit the body. const DeclRefExpr *LoopVarRef = S->getLoopVarRef(); LValue LCVal = EmitLValue(LoopVarRef); - Address LoopVarAddress = LCVal.getAddress(*this); + Address LoopVarAddress = LCVal.getAddress(); emitCapturedStmtCall(*this, LoopVarClosure, - {LoopVarAddress.getPointer(), IndVar}); + {LoopVarAddress.emitRawPointer(*this), IndVar}); RunCleanupsScope BodyScope(*this); EmitStmt(BodyStmt); @@ -2200,7 +2228,7 @@ void CodeGenFunction::EmitOMPLinearClauseFinal( DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); - Address OrigAddr = EmitLValue(&DRE).getAddress(*this); + Address OrigAddr = EmitLValue(&DRE).getAddress(); CodeGenFunction::OMPPrivateScope VarScope(*this); VarScope.addPrivate(OrigVD, OrigAddr); (void)VarScope.Privatize(); @@ -2267,7 +2295,7 @@ void CodeGenFunction::EmitOMPPrivateLoopCounters( DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD), E->getType(), VK_LValue, E->getExprLoc()); - (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress(*this)); + (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress()); } else { (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress()); } @@ -2433,13 +2461,12 @@ void CodeGenFunction::EmitOMPSimdFinal( } Address OrigAddr = Address::invalid(); if (CED) { - OrigAddr = - EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this); + OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(); } else { DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD), /*RefersToEnclosingVariableOrCapture=*/false, (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc()); - OrigAddr = EmitLValue(&DRE).getAddress(*this); + OrigAddr = EmitLValue(&DRE).getAddress(); } OMPPrivateScope VarScope(*this); VarScope.addPrivate(OrigVD, OrigAddr); @@ -2740,6 +2767,19 @@ void CodeGenFunction::EmitOMPTileDirective(const OMPTileDirective &S) { EmitStmt(S.getTransformedStmt()); } +void CodeGenFunction::EmitOMPReverseDirective(const OMPReverseDirective &S) { + // Emit the de-sugared statement. + OMPTransformDirectiveScopeRAII ReverseScope(*this, &S); + EmitStmt(S.getTransformedStmt()); +} + +void CodeGenFunction::EmitOMPInterchangeDirective( + const OMPInterchangeDirective &S) { + // Emit the de-sugared statement. + OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S); + EmitStmt(S.getTransformedStmt()); +} + void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) { bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder; @@ -2910,10 +2950,10 @@ void CodeGenFunction::EmitOMPOuterLoop( EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. - auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { + auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) { if (!DynamicOrOrdered) CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), - S.getDirectiveKind()); + LoopArgs.DKind); }; OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); } @@ -2963,12 +3003,14 @@ void CodeGenFunction::EmitOMPForOuterLoop( // run-sched-var ICV. If the ICV is set to auto, the schedule is // implementation defined // + // __kmpc_dispatch_init(); // while(__kmpc_dispatch_next(&LB, &UB)) { // idx = LB; // while (idx <= UB) { BODY; ++idx; // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. // } // inner loop // } + // __kmpc_dispatch_deinit(); // // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(static, chunk_size) is specified, iterations are divided into @@ -3019,8 +3061,12 @@ void CodeGenFunction::EmitOMPForOuterLoop( OuterLoopArgs.Cond = S.getCond(); OuterLoopArgs.NextLB = S.getNextLowerBound(); OuterLoopArgs.NextUB = S.getNextUpperBound(); + OuterLoopArgs.DKind = LoopArgs.DKind; EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs, emitOMPLoopBodyWithStopPoint, CodeGenOrdered); + if (DynamicOrOrdered) { + RT.emitForDispatchDeinit(*this, S.getBeginLoc()); + } } static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, @@ -3080,6 +3126,7 @@ void CodeGenFunction::EmitOMPDistributeOuterLoop( OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedNextUpperBound() : S.getNextUpperBound(); + OuterLoopArgs.DKind = OMPD_distribute; EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S, LoopScope, OuterLoopArgs, CodeGenLoopContent, @@ -3153,16 +3200,14 @@ static void emitDistributeParallelForDistributeInnerBoundParams( const auto &Dir = cast<OMPLoopDirective>(S); LValue LB = CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); - llvm::Value *LBCast = - CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), - CGF.SizeTy, /*isSigned=*/false); + llvm::Value *LBCast = CGF.Builder.CreateIntCast( + CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false); CapturedVars.push_back(LBCast); LValue UB = CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); - llvm::Value *UBCast = - CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), - CGF.SizeTy, /*isSigned=*/false); + llvm::Value *UBCast = CGF.Builder.CreateIntCast( + CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false); CapturedVars.push_back(UBCast); } @@ -3414,8 +3459,8 @@ bool CodeGenFunction::EmitOMPWorksharingLoop( // one chunk is distributed to each thread. Note that the size of // the chunks is unspecified in this case. CGOpenMPRuntime::StaticRTInput StaticInit( - IVSize, IVSigned, Ordered, IL.getAddress(CGF), - LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), + IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(), + UB.getAddress(), ST.getAddress(), StaticChunkedOne ? Chunk : nullptr); CGF.CGM.getOpenMPRuntime().emitForStaticInit( CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, @@ -3452,15 +3497,16 @@ bool CodeGenFunction::EmitOMPWorksharingLoop( // Tell the runtime we are done. auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), - S.getDirectiveKind()); + OMPD_for); }; OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); } else { // Emit the outer loop, which requests its work chunk [LB..UB] from // runtime and runs the inner loop to process it. - const OMPLoopArguments LoopArguments( - LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), - IL.getAddress(*this), Chunk, EUB); + OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(), + ST.getAddress(), IL.getAddress(), Chunk, + EUB); + LoopArguments.DKind = OMPD_for; EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered, LoopArguments, CGDispatchBounds); } @@ -3626,11 +3672,10 @@ static void emitScanBasedDirectiveFinals( RValue::get(OMPLast)); LValue DestLVal = CGF.EmitLValue(OrigExpr); LValue SrcLVal = CGF.EmitLValue(CopyArrayElem); - CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF), - SrcLVal.getAddress(CGF), - cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), - cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), - CopyOps[I]); + CGF.EmitOMPCopy( + PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(), + cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]); } } @@ -3740,7 +3785,7 @@ static void emitScanBasedDirective( cast<OpaqueValueExpr>( cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()), RValue::get(IVal)); - LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); + LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(); } PrivScope.addPrivate(LHSVD, LHSAddr); Address RHSAddr = Address::invalid(); @@ -3751,7 +3796,7 @@ static void emitScanBasedDirective( cast<OpaqueValueExpr>( cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()), RValue::get(OffsetIVal)); - RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); + RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(); } PrivScope.addPrivate(RHSVD, RHSAddr); ++ILHS; @@ -4065,8 +4110,8 @@ void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { OpenMPScheduleTy ScheduleKind; ScheduleKind.Schedule = OMPC_SCHEDULE_static; CGOpenMPRuntime::StaticRTInput StaticInit( - /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), - LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); + /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), + LB.getAddress(), UB.getAddress(), ST.getAddress()); CGF.CGM.getOpenMPRuntime().emitForStaticInit( CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); // UB = min(UB, GlobalUB); @@ -4082,7 +4127,7 @@ void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { // Tell the runtime we are done. auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), - S.getDirectiveKind()); + OMPD_sections); }; CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); @@ -4746,10 +4791,10 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue, CGF.Builder, false); - llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back(); // Get the call dbg.declare instruction we just created and update // its DIExpression to add offset to base address. - if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) { + auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare, + unsigned Offset) { SmallVector<uint64_t, 8> Ops; // Add offset to the base address if non zero. if (Offset) { @@ -4757,9 +4802,21 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( Ops.push_back(Offset); } Ops.push_back(llvm::dwarf::DW_OP_deref); - auto &Ctx = DDI->getContext(); - llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops); - Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr)); + Declare->setExpression(llvm::DIExpression::get(Ctx, Ops)); + }; + llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back(); + if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) + UpdateExpr(DDI->getContext(), DDI, Offset); + // If we're emitting using the new debug info format into a block + // without a terminator, the record will be "trailing". + assert(!Last.isTerminator() && "unexpected terminator"); + if (auto *Marker = + CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) { + for (llvm::DbgVariableRecord &DVR : llvm::reverse( + llvm::filterDbgVars(Marker->getDbgRecordRange()))) { + UpdateExpr(Last.getContext(), &DVR, Offset); + break; + } } } } @@ -4780,7 +4837,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.PrivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = CGF.CreateMemTemp( + RawAddress PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); @@ -4788,7 +4845,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4798,7 +4855,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".lastpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4811,7 +4868,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( Ty = CGF.getContext().getPointerType(Ty); if (isAllocatableDecl(VD)) Ty = CGF.getContext().getPointerType(Ty); - Address PrivatePtr = CGF.CreateMemTemp( + RawAddress PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); auto Result = UntiedLocalVars.insert( std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid()))); @@ -4833,7 +4890,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, Pair.second->getType(), VK_LValue, Pair.second->getExprLoc()); - Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF)); + Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress()); } for (const auto &Pair : PrivatePtrs) { Address Replacement = Address( @@ -4844,7 +4901,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( if (auto *DI = CGF.getDebugInfo()) if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) (void)DI->EmitDeclareOfAutoVariable( - Pair.first, Pair.second.getPointer(), CGF.Builder, + Pair.first, Pair.second.getBasePointer(), CGF.Builder, /*UsePointerValue*/ true); } // Adjust mapping for internal locals by mapping actual memory instead of @@ -4897,14 +4954,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = - Address(CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = Address( + CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), + CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -4955,7 +5012,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, + Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5074,7 +5131,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( // If there is no user-defined mapper, the mapper array will be nullptr. In // this case, we don't need to privatize it. if (!isa_and_nonnull<llvm::ConstantPointerNull>( - InputInfo.MappersArray.getPointer())) { + InputInfo.MappersArray.emitRawPointer(*this))) { MVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc()); TargetScope.addPrivate(MVD, InputInfo.MappersArray); @@ -5100,7 +5157,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -5179,14 +5236,14 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = - Address(CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = Address( + CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), + CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -5232,7 +5289,7 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, + Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5379,7 +5436,7 @@ void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) { Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end()); Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause( *this, Dependencies, DC->getBeginLoc()); - EmitStoreOfScalar(DepAddr.getPointer(), DOLVal); + EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal); return; } if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) { @@ -5480,8 +5537,8 @@ void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) { *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl())); LValue DestLVal = EmitLValue(TempExpr); LValue SrcLVal = EmitLValue(LHSs[I]); - EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this), - SrcLVal.getAddress(*this), + EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(), + SrcLVal.getAddress(), cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]); @@ -5502,11 +5559,10 @@ void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) { DestLVal = EmitLValue(RHSs[I]); SrcLVal = EmitLValue(TempExpr); } - EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this), - SrcLVal.getAddress(*this), - cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), - cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), - CopyOps[I]); + EmitOMPCopy( + PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(), + cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]); } } EmitBranch(IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock); @@ -5539,11 +5595,10 @@ void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) { RValue::get(IdxVal)); LValue DestLVal = EmitLValue(CopyArrayElem); LValue SrcLVal = EmitLValue(OrigExpr); - EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this), - SrcLVal.getAddress(*this), - cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), - cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), - CopyOps[I]); + EmitOMPCopy( + PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(), + cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]); } } EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock()); @@ -5581,11 +5636,10 @@ void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) { RValue::get(IdxVal)); LValue SrcLVal = EmitLValue(CopyArrayElem); LValue DestLVal = EmitLValue(OrigExpr); - EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this), - SrcLVal.getAddress(*this), - cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), - cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), - CopyOps[I]); + EmitOMPCopy( + PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(), + cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]); } if (!IsInclusive) { EmitBlock(ExclusiveExitBB); @@ -5710,8 +5764,8 @@ void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S, /* Chunked */ Chunk != nullptr) || StaticChunked) { CGOpenMPRuntime::StaticRTInput StaticInit( - IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this), - LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), + IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(), + LB.getAddress(), UB.getAddress(), ST.getAddress(), StaticChunked ? Chunk : nullptr); RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit); @@ -5782,13 +5836,13 @@ void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S, }); EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. - RT.emitForStaticFinish(*this, S.getEndLoc(), S.getDirectiveKind()); + RT.emitForStaticFinish(*this, S.getEndLoc(), OMPD_distribute); } else { // Emit the outer loop, which requests its work chunk [LB..UB] from // runtime and runs the inner loop to process it. const OMPLoopArguments LoopArguments = { - LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), - IL.getAddress(*this), Chunk}; + LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), + Chunk}; EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments, CodeGenLoop); } @@ -6102,8 +6156,7 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, // target platform. if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) && - (Update.getScalarVal()->getType() != - X.getAddress(CGF).getElementType())) || + (Update.getScalarVal()->getType() != X.getAddress().getElementType())) || !Context.getTargetInfo().hasBuiltinAtomic( Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) return std::make_pair(false, RValue::get(nullptr)); @@ -6119,10 +6172,10 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, }; if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) || - !CheckAtomicSupport(X.getAddress(CGF).getElementType(), BO)) + !CheckAtomicSupport(X.getAddress().getElementType(), BO)) return std::make_pair(false, RValue::get(nullptr)); - bool IsInteger = X.getAddress(CGF).getElementType()->isIntegerTy(); + bool IsInteger = X.getAddress().getElementType()->isIntegerTy(); llvm::AtomicRMWInst::BinOp RMWOp; switch (BO) { case BO_Add: @@ -6199,14 +6252,14 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { if (IsInteger) UpdateVal = CGF.Builder.CreateIntCast( - IC, X.getAddress(CGF).getElementType(), + IC, X.getAddress().getElementType(), X.getType()->hasSignedIntegerRepresentation()); else UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC, - X.getAddress(CGF).getElementType()); + X.getAddress().getElementType()); } llvm::Value *Res = - CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(CGF), UpdateVal, AO); + CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO); return std::make_pair(true, RValue::get(Res)); } @@ -6431,7 +6484,7 @@ static void emitOMPAtomicCompareExpr( } LValue XLVal = CGF.EmitLValue(X); - Address XAddr = XLVal.getAddress(CGF); + Address XAddr = XLVal.getAddress(); auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) { if (X->getType() == E->getType()) @@ -6447,36 +6500,36 @@ static void emitOMPAtomicCompareExpr( llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr; if (auto *CI = dyn_cast<llvm::ConstantInt>(EVal)) EVal = CGF.Builder.CreateIntCast( - CI, XLVal.getAddress(CGF).getElementType(), + CI, XLVal.getAddress().getElementType(), E->getType()->hasSignedIntegerRepresentation()); if (DVal) if (auto *CI = dyn_cast<llvm::ConstantInt>(DVal)) DVal = CGF.Builder.CreateIntCast( - CI, XLVal.getAddress(CGF).getElementType(), + CI, XLVal.getAddress().getElementType(), D->getType()->hasSignedIntegerRepresentation()); llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{ - XAddr.getPointer(), XAddr.getElementType(), + XAddr.emitRawPointer(CGF), XAddr.getElementType(), X->getType()->hasSignedIntegerRepresentation(), X->getType().isVolatileQualified()}; llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal; if (V) { LValue LV = CGF.EmitLValue(V); - Address Addr = LV.getAddress(CGF); - VOpVal = {Addr.getPointer(), Addr.getElementType(), + Address Addr = LV.getAddress(); + VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), V->getType()->hasSignedIntegerRepresentation(), V->getType().isVolatileQualified()}; } if (R) { LValue LV = CGF.EmitLValue(R); - Address Addr = LV.getAddress(CGF); - ROpVal = {Addr.getPointer(), Addr.getElementType(), + Address Addr = LV.getAddress(); + ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), R->getType()->hasSignedIntegerRepresentation(), R->getType().isVolatileQualified()}; } if (FailAO == llvm::AtomicOrdering::NotAtomic) { - // fail clause was not mentionend on the + // fail clause was not mentioned on the // "#pragma omp atomic compare" construct. CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare( CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr, @@ -6520,7 +6573,7 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, } void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { - llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic; + llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering(); // Fail Memory Clause Ordering. llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic; bool MemOrderingSpecified = false; @@ -6546,6 +6599,9 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause, // if it is first). OpenMPClauseKind K = C->getClauseKind(); + // TBD + if (K == OMPC_weak) + return; if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire || K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint) continue; @@ -7011,7 +7067,7 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) { std::tie(NumDependences, DependenciesArray) = CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences, S.getBeginLoc()); - DependenceList = DependenciesArray.getPointer(); + DependenceList = DependenciesArray.emitRawPointer(*this); } Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>(); @@ -7020,31 +7076,47 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) { S.getSingleClause<OMPUseClause>())) && "OMPNowaitClause clause is used separately in OMPInteropDirective."); - if (const auto *C = S.getSingleClause<OMPInitClause>()) { - llvm::Value *InteropvarPtr = - EmitLValue(C->getInteropVar()).getPointer(*this); - llvm::omp::OMPInteropType InteropType = llvm::omp::OMPInteropType::Unknown; - if (C->getIsTarget()) { - InteropType = llvm::omp::OMPInteropType::Target; - } else { - assert(C->getIsTargetSync() && "Expected interop-type target/targetsync"); - InteropType = llvm::omp::OMPInteropType::TargetSync; + auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>(); + if (!ItOMPInitClause.empty()) { + // Look at the multiple init clauses + for (const OMPInitClause *C : ItOMPInitClause) { + llvm::Value *InteropvarPtr = + EmitLValue(C->getInteropVar()).getPointer(*this); + llvm::omp::OMPInteropType InteropType = + llvm::omp::OMPInteropType::Unknown; + if (C->getIsTarget()) { + InteropType = llvm::omp::OMPInteropType::Target; + } else { + assert(C->getIsTargetSync() && + "Expected interop-type target/targetsync"); + InteropType = llvm::omp::OMPInteropType::TargetSync; + } + OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType, + Device, NumDependences, DependenceList, + Data.HasNowaitClause); + } + } + auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>(); + if (!ItOMPDestroyClause.empty()) { + // Look at the multiple destroy clauses + for (const OMPDestroyClause *C : ItOMPDestroyClause) { + llvm::Value *InteropvarPtr = + EmitLValue(C->getInteropVar()).getPointer(*this); + OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device, + NumDependences, DependenceList, + Data.HasNowaitClause); + } + } + auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>(); + if (!ItOMPUseClause.empty()) { + // Look at the multiple use clauses + for (const OMPUseClause *C : ItOMPUseClause) { + llvm::Value *InteropvarPtr = + EmitLValue(C->getInteropVar()).getPointer(*this); + OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device, + NumDependences, DependenceList, + Data.HasNowaitClause); } - OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType, Device, - NumDependences, DependenceList, - Data.HasNowaitClause); - } else if (const auto *C = S.getSingleClause<OMPDestroyClause>()) { - llvm::Value *InteropvarPtr = - EmitLValue(C->getInteropVar()).getPointer(*this); - OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device, - NumDependences, DependenceList, - Data.HasNowaitClause); - } else if (const auto *C = S.getSingleClause<OMPUseClause>()) { - llvm::Value *InteropvarPtr = - EmitLValue(C->getInteropVar()).getPointer(*this); - OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device, - NumDependences, DependenceList, - Data.HasNowaitClause); } } @@ -7245,7 +7317,7 @@ void CodeGenFunction::EmitOMPUseDevicePtrClause( static const VarDecl *getBaseDecl(const Expr *Ref) { const Expr *Base = Ref->IgnoreParenImpCasts(); - while (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Base)) + while (const auto *OASE = dyn_cast<ArraySectionExpr>(Base)) Base = OASE->getBase()->IgnoreParenImpCasts(); while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Base)) Base = ASE->getBase()->IgnoreParenImpCasts(); @@ -7871,7 +7943,7 @@ void CodeGenFunction::EmitOMPGenericLoopDirective( void CodeGenFunction::EmitOMPParallelGenericLoopDirective( const OMPLoopDirective &S) { - // Emit combined directive as if its consituent constructs are 'parallel' + // Emit combined directive as if its constituent constructs are 'parallel' // and 'for'. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); @@ -7891,11 +7963,9 @@ void CodeGenFunction::EmitOMPParallelGenericLoopDirective( void CodeGenFunction::EmitOMPTeamsGenericLoopDirective( const OMPTeamsGenericLoopDirective &S) { // To be consistent with current behavior of 'target teams loop', emit - // 'teams loop' as if its constituent constructs are 'distribute, - // 'parallel, and 'for'. + // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { - CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, - S.getDistInc()); + CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. @@ -7909,15 +7979,33 @@ void CodeGenFunction::EmitOMPTeamsGenericLoopDirective( CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; - emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); + emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } -static void -emitTargetTeamsGenericLoopRegion(CodeGenFunction &CGF, - const OMPTargetTeamsGenericLoopDirective &S, - PrePostActionTy &Action) { +#ifndef NDEBUG +static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, + std::string StatusMsg, + const OMPExecutableDirective &D) { + bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice; + if (IsDevice) + StatusMsg += ": DEVICE"; + else + StatusMsg += ": HOST"; + SourceLocation L = D.getBeginLoc(); + auto &SM = CGF.getContext().getSourceManager(); + PresumedLoc PLoc = SM.getPresumedLoc(L); + const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr; + unsigned LineNo = + PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L); + llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n"; +} +#endif + +static void emitTargetTeamsGenericLoopRegionAsParallel( + CodeGenFunction &CGF, PrePostActionTy &Action, + const OMPTargetTeamsGenericLoopDirective &S) { Action.Enter(CGF); // Emit 'teams loop' as if its constituent constructs are 'distribute, // 'parallel, and 'for'. @@ -7937,19 +8025,50 @@ emitTargetTeamsGenericLoopRegion(CodeGenFunction &CGF, CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; - + DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE, + emitTargetTeamsLoopCodegenStatus( + CGF, TTL_CODEGEN_TYPE " as parallel for", S)); emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } -/// Emit combined directive 'target teams loop' as if its constituent -/// constructs are 'target', 'teams', 'distribute', 'parallel', and 'for'. +static void emitTargetTeamsGenericLoopRegionAsDistribute( + CodeGenFunction &CGF, PrePostActionTy &Action, + const OMPTargetTeamsGenericLoopDirective &S) { + Action.Enter(CGF); + // Emit 'teams loop' as if its constituent construct is 'distribute'. + auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); + }; + + // Emit teams region as a standalone region. + auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, + PrePostActionTy &Action) { + Action.Enter(CGF); + CodeGenFunction::OMPPrivateScope PrivateScope(CGF); + CGF.EmitOMPReductionClauseInit(S, PrivateScope); + (void)PrivateScope.Privatize(); + CGF.CGM.getOpenMPRuntime().emitInlinedDirective( + CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); + CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); + }; + DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE, + emitTargetTeamsLoopCodegenStatus( + CGF, TTL_CODEGEN_TYPE " as distribute", S)); + emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); + emitPostUpdateForReductionClause(CGF, S, + [](CodeGenFunction &) { return nullptr; }); +} + void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDirective( const OMPTargetTeamsGenericLoopDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { - emitTargetTeamsGenericLoopRegion(CGF, S, Action); + if (S.canBeParallelFor()) + emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S); + else + emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } @@ -7959,7 +8078,10 @@ void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction( const OMPTargetTeamsGenericLoopDirective &S) { // Emit SPMD target parallel loop region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { - emitTargetTeamsGenericLoopRegion(CGF, S, Action); + if (S.canBeParallelFor()) + emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S); + else + emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; @@ -8033,7 +8155,7 @@ void CodeGenFunction::EmitSimpleOMPExecutableDirective( continue; if (!CGF.LocalDeclMap.count(VD)) { LValue GlobLVal = CGF.EmitLValue(Ref); - GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF)); + GlobalsScope.addPrivate(VD, GlobLVal.getAddress()); } } } @@ -8048,7 +8170,7 @@ void CodeGenFunction::EmitSimpleOMPExecutableDirective( const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { LValue GlobLVal = CGF.EmitLValue(E); - GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF)); + GlobalsScope.addPrivate(VD, GlobLVal.getAddress()); } if (isa<OMPCapturedExprDecl>(VD)) { // Emit only those that were not explicitly referenced in clauses. diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGVTT.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGVTT.cpp index 1d3f14f1c534..20bd2c2fc2c6 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGVTT.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGVTT.cpp @@ -77,9 +77,23 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.AddressPointIndex), }; + // Add inrange attribute to indicate that only the VTableIndex can be + // accessed. + unsigned ComponentSize = + CGM.getDataLayout().getTypeAllocSize(getVTableComponentType()); + unsigned VTableSize = CGM.getDataLayout().getTypeAllocSize( + cast<llvm::StructType>(VTable->getValueType()) + ->getElementType(AddressPoint.VTableIndex)); + unsigned Offset = ComponentSize * AddressPoint.AddressPointIndex; + llvm::ConstantRange InRange(llvm::APInt(32, -Offset, true), + llvm::APInt(32, VTableSize - Offset, true)); llvm::Constant *Init = llvm::ConstantExpr::getGetElementPtr( - VTable->getValueType(), VTable, Idxs, /*InBounds=*/true, - /*InRangeIndex=*/1); + VTable->getValueType(), VTable, Idxs, /*InBounds=*/true, InRange); + + if (const auto &Schema = + CGM.getCodeGenOpts().PointerAuth.CXXVTTVTablePointers) + Init = CGM.getConstantSignedPointer(Init, Schema, nullptr, GlobalDecl(), + QualType()); VTTComponents.push_back(Init); } @@ -129,23 +143,24 @@ uint64_t CodeGenVTables::getSubVTTIndex(const CXXRecordDecl *RD, BaseSubobject Base) { BaseSubobjectPairTy ClassSubobjectPair(RD, Base); - SubVTTIndiciesMapTy::iterator I = SubVTTIndicies.find(ClassSubobjectPair); - if (I != SubVTTIndicies.end()) + SubVTTIndicesMapTy::iterator I = SubVTTIndices.find(ClassSubobjectPair); + if (I != SubVTTIndices.end()) return I->second; VTTBuilder Builder(CGM.getContext(), RD, /*GenerateDefinition=*/false); - for (llvm::DenseMap<BaseSubobject, uint64_t>::const_iterator I = - Builder.getSubVTTIndicies().begin(), - E = Builder.getSubVTTIndicies().end(); I != E; ++I) { + for (llvm::DenseMap<BaseSubobject, uint64_t>::const_iterator + I = Builder.getSubVTTIndices().begin(), + E = Builder.getSubVTTIndices().end(); + I != E; ++I) { // Insert all indices. BaseSubobjectPairTy ClassSubobjectPair(RD, I->first); - SubVTTIndicies.insert(std::make_pair(ClassSubobjectPair, I->second)); + SubVTTIndices.insert(std::make_pair(ClassSubobjectPair, I->second)); } - I = SubVTTIndicies.find(ClassSubobjectPair); - assert(I != SubVTTIndicies.end() && "Did not find index!"); + I = SubVTTIndices.find(ClassSubobjectPair); + assert(I != SubVTTIndices.end() && "Did not find index!"); return I->second; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGVTables.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGVTables.cpp index 8dee3f74b44b..267bdf098297 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGVTables.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGVTables.cpp @@ -95,7 +95,7 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF, CGF, Address(ReturnValue, CGF.ConvertTypeForMem(ResultType->getPointeeType()), ClassAlign), - Thunk.Return); + ClassDecl, Thunk.Return); if (NullCheckValue) { CGF.Builder.CreateBr(AdjustEnd); @@ -131,6 +131,12 @@ static void resolveTopLevelMetadata(llvm::Function *Fn, // they are referencing. for (auto &BB : *Fn) { for (auto &I : BB) { + for (llvm::DbgVariableRecord &DVR : + llvm::filterDbgVars(I.getDbgRecordRange())) { + auto *DILocal = DVR.getVariable(); + if (!DILocal->isResolved()) + DILocal->resolve(); + } if (auto *DII = dyn_cast<llvm::DbgVariableIntrinsic>(&I)) { auto *DILocal = DII->getVariable(); if (!DILocal->isResolved()) @@ -201,21 +207,22 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn, // Find the first store of "this", which will be to the alloca associated // with "this". - Address ThisPtr = - Address(&*AI, ConvertTypeForMem(MD->getFunctionObjectParameterType()), - CGM.getClassPointerAlignment(MD->getParent())); + Address ThisPtr = makeNaturalAddressForPointer( + &*AI, MD->getFunctionObjectParameterType(), + CGM.getClassPointerAlignment(MD->getParent())); llvm::BasicBlock *EntryBB = &Fn->front(); llvm::BasicBlock::iterator ThisStore = llvm::find_if(*EntryBB, [&](llvm::Instruction &I) { - return isa<llvm::StoreInst>(I) && - I.getOperand(0) == ThisPtr.getPointer(); + return isa<llvm::StoreInst>(I) && I.getOperand(0) == &*AI; }); assert(ThisStore != EntryBB->end() && "Store of this should be in entry block?"); // Adjust "this", if necessary. Builder.SetInsertPoint(&*ThisStore); - llvm::Value *AdjustedThisPtr = - CGM.getCXXABI().performThisAdjustment(*this, ThisPtr, Thunk.This); + + const CXXRecordDecl *ThisValueClass = Thunk.ThisType->getPointeeCXXRecordDecl(); + llvm::Value *AdjustedThisPtr = CGM.getCXXABI().performThisAdjustment( + *this, ThisPtr, ThisValueClass, Thunk); AdjustedThisPtr = Builder.CreateBitCast(AdjustedThisPtr, ThisStore->getOperand(0)->getType()); ThisStore->setOperand(0, AdjustedThisPtr); @@ -302,10 +309,15 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::FunctionCallee Callee, const CXXMethodDecl *MD = cast<CXXMethodDecl>(CurGD.getDecl()); // Adjust the 'this' pointer if necessary + const CXXRecordDecl *ThisValueClass = + MD->getThisType()->getPointeeCXXRecordDecl(); + if (Thunk) + ThisValueClass = Thunk->ThisType->getPointeeCXXRecordDecl(); + llvm::Value *AdjustedThisPtr = - Thunk ? CGM.getCXXABI().performThisAdjustment( - *this, LoadCXXThisAddress(), Thunk->This) - : LoadCXXThis(); + Thunk ? CGM.getCXXABI().performThisAdjustment(*this, LoadCXXThisAddress(), + ThisValueClass, *Thunk) + : LoadCXXThis(); // If perfect forwarding is required a variadic method, a method using // inalloca, or an unprototyped thunk, use musttail. Emit an error if this @@ -499,10 +511,22 @@ llvm::Constant *CodeGenVTables::maybeEmitThunk(GlobalDecl GD, SmallString<256> Name; MangleContext &MCtx = CGM.getCXXABI().getMangleContext(); llvm::raw_svector_ostream Out(Name); - if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) - MCtx.mangleCXXDtorThunk(DD, GD.getDtorType(), TI.This, Out); - else - MCtx.mangleThunk(MD, TI, Out); + + if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) { + MCtx.mangleCXXDtorThunk(DD, GD.getDtorType(), TI, + /* elideOverrideInfo */ false, Out); + } else + MCtx.mangleThunk(MD, TI, /* elideOverrideInfo */ false, Out); + + if (CGM.getContext().useAbbreviatedThunkName(GD, Name.str())) { + Name = ""; + if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) + MCtx.mangleCXXDtorThunk(DD, GD.getDtorType(), TI, + /* elideOverrideInfo */ true, Out); + else + MCtx.mangleThunk(MD, TI, /* elideOverrideInfo */ true, Out); + } + llvm::Type *ThunkVTableTy = CGM.getTypes().GetFunctionTypeForVTable(GD); llvm::Constant *Thunk = CGM.GetAddrOfThunk(Name, ThunkVTableTy, GD); @@ -814,11 +838,17 @@ void CodeGenVTables::addVTableComponent(ConstantArrayBuilder &builder, nextVTableThunkIndex++; fnPtr = maybeEmitThunk(GD, thunkInfo, /*ForVTable=*/true); + if (CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers) { + assert(thunkInfo.Method && "Method not set"); + GD = GD.getWithDecl(thunkInfo.Method); + } // Otherwise we can use the method definition directly. } else { llvm::Type *fnTy = CGM.getTypes().GetFunctionTypeForVTable(GD); fnPtr = CGM.GetAddrOfFunction(GD, fnTy, /*ForVTable=*/true); + if (CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers) + GD = getItaniumVTableContext().findOriginalMethod(GD); } if (useRelativeLayout()) { @@ -836,6 +866,9 @@ void CodeGenVTables::addVTableComponent(ConstantArrayBuilder &builder, if (FnAS != GVAS) fnPtr = llvm::ConstantExpr::getAddrSpaceCast(fnPtr, CGM.GlobalsInt8PtrTy); + if (const auto &Schema = + CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers) + return builder.addSignedPointer(fnPtr, Schema, GD, QualType()); return builder.add(fnPtr); } } @@ -1045,29 +1078,41 @@ llvm::GlobalVariable::LinkageTypes CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { if (!RD->isExternallyVisible()) return llvm::GlobalVariable::InternalLinkage; - - // We're at the end of the translation unit, so the current key - // function is fully correct. - const CXXMethodDecl *keyFunction = Context.getCurrentKeyFunction(RD); - if (keyFunction && !RD->hasAttr<DLLImportAttr>()) { + + // In windows, the linkage of vtable is not related to modules. + bool IsInNamedModule = !getTarget().getCXXABI().isMicrosoft() && + RD->isInNamedModule(); + // If the CXXRecordDecl is not in a module unit, we need to get + // its key function. We're at the end of the translation unit, so the current + // key function is fully correct. + const CXXMethodDecl *keyFunction = + IsInNamedModule ? nullptr : Context.getCurrentKeyFunction(RD); + if (IsInNamedModule || (keyFunction && !RD->hasAttr<DLLImportAttr>())) { // If this class has a key function, use that to determine the // linkage of the vtable. const FunctionDecl *def = nullptr; - if (keyFunction->hasBody(def)) + if (keyFunction && keyFunction->hasBody(def)) keyFunction = cast<CXXMethodDecl>(def); - switch (keyFunction->getTemplateSpecializationKind()) { - case TSK_Undeclared: - case TSK_ExplicitSpecialization: + bool IsExternalDefinition = + IsInNamedModule ? RD->shouldEmitInExternalSource() : !def; + + TemplateSpecializationKind Kind = + IsInNamedModule ? RD->getTemplateSpecializationKind() + : keyFunction->getTemplateSpecializationKind(); + + switch (Kind) { + case TSK_Undeclared: + case TSK_ExplicitSpecialization: assert( - (def || CodeGenOpts.OptimizationLevel > 0 || + (IsInNamedModule || def || CodeGenOpts.OptimizationLevel > 0 || CodeGenOpts.getDebugInfo() != llvm::codegenoptions::NoDebugInfo) && - "Shouldn't query vtable linkage without key function, " - "optimizations, or debug info"); - if (!def && CodeGenOpts.OptimizationLevel > 0) + "Shouldn't query vtable linkage without the class in module units, " + "key function, optimizations, or debug info"); + if (IsExternalDefinition && CodeGenOpts.OptimizationLevel > 0) return llvm::GlobalVariable::AvailableExternallyLinkage; - if (keyFunction->isInlined()) + if (keyFunction && keyFunction->isInlined()) return !Context.getLangOpts().AppleKext ? llvm::GlobalVariable::LinkOnceODRLinkage : llvm::Function::InternalLinkage; @@ -1086,7 +1131,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { case TSK_ExplicitInstantiationDeclaration: llvm_unreachable("Should not have been asked to emit this"); - } + } } // -fapple-kext mode does not support weak linkage, so we must use @@ -1180,22 +1225,20 @@ bool CodeGenVTables::isVTableExternal(const CXXRecordDecl *RD) { TSK == TSK_ExplicitInstantiationDefinition) return false; + // Otherwise, if the class is attached to a module, the tables are uniquely + // emitted in the object for the module unit in which it is defined. + if (RD->isInNamedModule()) + return RD->shouldEmitInExternalSource(); + // Otherwise, if the class doesn't have a key function (possibly // anymore), the vtable must be defined here. const CXXMethodDecl *keyFunction = CGM.getContext().getCurrentKeyFunction(RD); if (!keyFunction) return false; - const FunctionDecl *Def; // Otherwise, if we don't have a definition of the key function, the // vtable must be defined somewhere else. - if (!keyFunction->hasBody(Def)) - return true; - - assert(Def && "The body of the key function is not assigned to Def?"); - // If the non-inline key function comes from another module unit, the vtable - // must be defined there. - return Def->isInAnotherModuleUnit() && !Def->isInlineSpecified(); + return !keyFunction->hasBody(); } /// Given that we're currently at the end of the translation unit, and diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGVTables.h b/contrib/llvm-project/clang/lib/CodeGen/CGVTables.h index 9d4223547050..c06bf7a525d9 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGVTables.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGVTables.h @@ -38,10 +38,10 @@ class CodeGenVTables { typedef VTableLayout::AddressPointsMapTy VTableAddressPointsMapTy; typedef std::pair<const CXXRecordDecl *, BaseSubobject> BaseSubobjectPairTy; - typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SubVTTIndiciesMapTy; + typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SubVTTIndicesMapTy; - /// SubVTTIndicies - Contains indices into the various sub-VTTs. - SubVTTIndiciesMapTy SubVTTIndicies; + /// SubVTTIndices - Contains indices into the various sub-VTTs. + SubVTTIndicesMapTy SubVTTIndices; typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SecondaryVirtualPointerIndicesMapTy; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGValue.h b/contrib/llvm-project/clang/lib/CodeGen/CGValue.h index 1e6f67250583..c4ec8d207d2e 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGValue.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CGValue.h @@ -14,12 +14,14 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGVALUE_H #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H +#include "Address.h" +#include "CGPointerAuthInfo.h" +#include "CodeGenTBAA.h" +#include "EHScopeStack.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Type.h" -#include "llvm/IR/Value.h" #include "llvm/IR/Type.h" -#include "Address.h" -#include "CodeGenTBAA.h" +#include "llvm/IR/Value.h" namespace llvm { class Constant; @@ -28,57 +30,64 @@ namespace llvm { namespace clang { namespace CodeGen { - class AggValueSlot; - class CodeGenFunction; - struct CGBitFieldInfo; +class AggValueSlot; +class CGBuilderTy; +class CodeGenFunction; +struct CGBitFieldInfo; /// RValue - This trivial value class is used to represent the result of an /// expression that is evaluated. It can be one of three things: either a /// simple LLVM SSA value, a pair of SSA values for complex numbers, or the /// address of an aggregate value in memory. class RValue { - enum Flavor { Scalar, Complex, Aggregate }; + friend struct DominatingValue<RValue>; - // The shift to make to an aggregate's alignment to make it look - // like a pointer. - enum { AggAlignShift = 4 }; + enum FlavorEnum { Scalar, Complex, Aggregate }; - // Stores first value and flavor. - llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1; - // Stores second value and volatility. - llvm::PointerIntPair<llvm::Value *, 1, bool> V2; - // Stores element type for aggregate values. - llvm::Type *ElementType; + union { + // Stores first and second value. + struct { + llvm::Value *first; + llvm::Value *second; + } Vals; + + // Stores aggregate address. + Address AggregateAddr; + }; + + unsigned IsVolatile : 1; + unsigned Flavor : 2; public: - bool isScalar() const { return V1.getInt() == Scalar; } - bool isComplex() const { return V1.getInt() == Complex; } - bool isAggregate() const { return V1.getInt() == Aggregate; } + RValue() : Vals{nullptr, nullptr}, Flavor(Scalar) {} + + bool isScalar() const { return Flavor == Scalar; } + bool isComplex() const { return Flavor == Complex; } + bool isAggregate() const { return Flavor == Aggregate; } - bool isVolatileQualified() const { return V2.getInt(); } + bool isVolatileQualified() const { return IsVolatile; } /// getScalarVal() - Return the Value* of this scalar value. llvm::Value *getScalarVal() const { assert(isScalar() && "Not a scalar!"); - return V1.getPointer(); + return Vals.first; } /// getComplexVal - Return the real/imag components of this complex value. /// std::pair<llvm::Value *, llvm::Value *> getComplexVal() const { - return std::make_pair(V1.getPointer(), V2.getPointer()); + return std::make_pair(Vals.first, Vals.second); } /// getAggregateAddr() - Return the Value* of the address of the aggregate. Address getAggregateAddress() const { assert(isAggregate() && "Not an aggregate!"); - auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift; - return Address( - V1.getPointer(), ElementType, CharUnits::fromQuantity(align)); + return AggregateAddr; } - llvm::Value *getAggregatePointer() const { - assert(isAggregate() && "Not an aggregate!"); - return V1.getPointer(); + + llvm::Value *getAggregatePointer(QualType PointeeType, + CodeGenFunction &CGF) const { + return getAggregateAddress().getBasePointer(); } static RValue getIgnored() { @@ -88,17 +97,19 @@ public: static RValue get(llvm::Value *V) { RValue ER; - ER.V1.setPointer(V); - ER.V1.setInt(Scalar); - ER.V2.setInt(false); + ER.Vals.first = V; + ER.Flavor = Scalar; + ER.IsVolatile = false; return ER; } + static RValue get(Address Addr, CodeGenFunction &CGF) { + return RValue::get(Addr.emitRawPointer(CGF)); + } static RValue getComplex(llvm::Value *V1, llvm::Value *V2) { RValue ER; - ER.V1.setPointer(V1); - ER.V2.setPointer(V2); - ER.V1.setInt(Complex); - ER.V2.setInt(false); + ER.Vals = {V1, V2}; + ER.Flavor = Complex; + ER.IsVolatile = false; return ER; } static RValue getComplex(const std::pair<llvm::Value *, llvm::Value *> &C) { @@ -107,15 +118,15 @@ public: // FIXME: Aggregate rvalues need to retain information about whether they are // volatile or not. Remove default to find all places that probably get this // wrong. + + /// Convert an Address to an RValue. If the Address is not + /// signed, create an RValue using the unsigned address. Otherwise, resign the + /// address using the provided type. static RValue getAggregate(Address addr, bool isVolatile = false) { RValue ER; - ER.V1.setPointer(addr.getPointer()); - ER.V1.setInt(Aggregate); - ER.ElementType = addr.getElementType(); - - auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity()); - ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift)); - ER.V2.setInt(isVolatile); + ER.AggregateAddr = addr; + ER.Flavor = Aggregate; + ER.IsVolatile = isVolatile; return ER; } }; @@ -178,8 +189,10 @@ class LValue { MatrixElt // This is a matrix element, use getVector* } LVType; - llvm::Value *V; - llvm::Type *ElementType; + union { + Address Addr = Address::invalid(); + llvm::Value *V; + }; union { // Index into a vector subscript: V[i] @@ -197,10 +210,6 @@ class LValue { // 'const' is unused here Qualifiers Quals; - // The alignment to use when accessing this lvalue. (For vector elements, - // this is the alignment of the whole vector.) - unsigned Alignment; - // objective-c's ivar bool Ivar:1; @@ -225,32 +234,25 @@ class LValue { // this lvalue. bool Nontemporal : 1; - // The pointer is known not to be null. - bool IsKnownNonNull : 1; - LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Expr *BaseIvarExp; private: - void Initialize(QualType Type, Qualifiers Quals, CharUnits Alignment, + void Initialize(QualType Type, Qualifiers Quals, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - assert((!Alignment.isZero() || Type->isIncompleteType()) && - "initializing l-value with zero alignment!"); - if (isGlobalReg()) - assert(ElementType == nullptr && "Global reg does not store elem type"); - else - assert(ElementType != nullptr && "Must have elem type"); - this->Type = Type; this->Quals = Quals; const unsigned MaxAlign = 1U << 31; - this->Alignment = Alignment.getQuantity() <= MaxAlign - ? Alignment.getQuantity() - : MaxAlign; - assert(this->Alignment == Alignment.getQuantity() && - "Alignment exceeds allowed max!"); + CharUnits Alignment = Addr.getAlignment(); + assert((isGlobalReg() || !Alignment.isZero() || Type->isIncompleteType()) && + "initializing l-value with zero alignment!"); + if (Alignment.getQuantity() > MaxAlign) { + assert(false && "Alignment exceeds allowed max!"); + Alignment = CharUnits::fromQuantity(MaxAlign); + } + this->Addr = Addr; this->BaseInfo = BaseInfo; this->TBAAInfo = TBAAInfo; @@ -262,6 +264,16 @@ private: this->BaseIvarExp = nullptr; } + void initializeSimpleLValue(Address Addr, QualType Type, + LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, + ASTContext &Context) { + Qualifiers QS = Type.getQualifiers(); + QS.setObjCGCAttr(Context.getObjCGCAttrKind(Type)); + LVType = Simple; + Initialize(Type, QS, Addr, BaseInfo, TBAAInfo); + assert(Addr.getBasePointer()->getType()->isPointerTy()); + } + public: bool isSimple() const { return LVType == Simple; } bool isVectorElt() const { return LVType == VectorElt; } @@ -328,45 +340,44 @@ public: LangAS getAddressSpace() const { return Quals.getAddressSpace(); } - CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); } - void setAlignment(CharUnits A) { Alignment = A.getQuantity(); } + CharUnits getAlignment() const { return Addr.getAlignment(); } + void setAlignment(CharUnits A) { Addr.setAlignment(A); } LValueBaseInfo getBaseInfo() const { return BaseInfo; } void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; } - KnownNonNull_t isKnownNonNull() const { - return (KnownNonNull_t)IsKnownNonNull; - } + KnownNonNull_t isKnownNonNull() const { return Addr.isKnownNonNull(); } LValue setKnownNonNull() { - IsKnownNonNull = true; + Addr.setKnownNonNull(); return *this; } // simple lvalue - llvm::Value *getPointer(CodeGenFunction &CGF) const { - assert(isSimple()); - return V; - } - Address getAddress(CodeGenFunction &CGF) const { - return Address(getPointer(CGF), ElementType, getAlignment(), - isKnownNonNull()); - } - void setAddress(Address address) { - assert(isSimple()); - V = address.getPointer(); - ElementType = address.getElementType(); - Alignment = address.getAlignment().getQuantity(); - IsKnownNonNull = address.isKnownNonNull(); + llvm::Value *getPointer(CodeGenFunction &CGF) const; + llvm::Value *emitResignedPointer(QualType PointeeTy, + CodeGenFunction &CGF) const; + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const; + + Address getAddress() const { return Addr; } + + void setAddress(Address address) { Addr = address; } + + CGPointerAuthInfo getPointerAuthInfo() const { + return Addr.getPointerAuthInfo(); } // vector elt lvalue Address getVectorAddress() const { - return Address(getVectorPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isVectorElt()); + return Addr; + } + llvm::Value *getRawVectorPointer(CodeGenFunction &CGF) const { + assert(isVectorElt()); + return Addr.emitRawPointer(CGF); } llvm::Value *getVectorPointer() const { assert(isVectorElt()); - return V; + return Addr.getBasePointer(); } llvm::Value *getVectorIdx() const { assert(isVectorElt()); @@ -374,12 +385,12 @@ public: } Address getMatrixAddress() const { - return Address(getMatrixPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isMatrixElt()); + return Addr; } llvm::Value *getMatrixPointer() const { assert(isMatrixElt()); - return V; + return Addr.getBasePointer(); } llvm::Value *getMatrixIdx() const { assert(isMatrixElt()); @@ -388,12 +399,12 @@ public: // extended vector elements. Address getExtVectorAddress() const { - return Address(getExtVectorPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isExtVectorElt()); + return Addr; } - llvm::Value *getExtVectorPointer() const { + llvm::Value *getRawExtVectorPointer(CodeGenFunction &CGF) const { assert(isExtVectorElt()); - return V; + return Addr.emitRawPointer(CGF); } llvm::Constant *getExtVectorElts() const { assert(isExtVectorElt()); @@ -402,10 +413,14 @@ public: // bitfield lvalue Address getBitFieldAddress() const { - return Address(getBitFieldPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isBitField()); + return Addr; + } + llvm::Value *getRawBitFieldPointer(CodeGenFunction &CGF) const { + assert(isBitField()); + return Addr.emitRawPointer(CGF); } - llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } + const CGBitFieldInfo &getBitFieldInfo() const { assert(isBitField()); return *BitFieldInfo; @@ -414,18 +429,13 @@ public: // global register lvalue llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; } - static LValue MakeAddr(Address address, QualType type, ASTContext &Context, + static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - Qualifiers qs = type.getQualifiers(); - qs.setObjCGCAttr(Context.getObjCGCAttrKind(type)); - LValue R; R.LVType = Simple; - assert(address.getPointer()->getType()->isPointerTy()); - R.V = address.getPointer(); - R.ElementType = address.getElementType(); - R.IsKnownNonNull = address.isKnownNonNull(); - R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo); + R.initializeSimpleLValue(Addr, type, BaseInfo, TBAAInfo, Context); + R.Addr = Addr; + assert(Addr.getType()->isPointerTy()); return R; } @@ -434,26 +444,18 @@ public: TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = VectorElt; - R.V = vecAddress.getPointer(); - R.ElementType = vecAddress.getElementType(); R.VectorIdx = Idx; - R.IsKnownNonNull = vecAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), vecAddress, BaseInfo, TBAAInfo); return R; } - static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, + static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = ExtVectorElt; - R.V = vecAddress.getPointer(); - R.ElementType = vecAddress.getElementType(); R.VectorElts = Elts; - R.IsKnownNonNull = vecAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); return R; } @@ -468,12 +470,8 @@ public: TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = BitField; - R.V = Addr.getPointer(); - R.ElementType = Addr.getElementType(); R.BitFieldInfo = &Info; - R.IsKnownNonNull = Addr.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo, - TBAAInfo); + R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); return R; } @@ -481,11 +479,9 @@ public: QualType type) { LValue R; R.LVType = GlobalReg; - R.V = V; - R.ElementType = nullptr; - R.IsKnownNonNull = true; - R.Initialize(type, type.getQualifiers(), alignment, + R.Initialize(type, type.getQualifiers(), Address::invalid(), LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); + R.V = V; return R; } @@ -494,17 +490,13 @@ public: TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = MatrixElt; - R.V = matAddress.getPointer(); - R.ElementType = matAddress.getElementType(); R.VectorIdx = Idx; - R.IsKnownNonNull = matAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), matAddress, BaseInfo, TBAAInfo); return R; } - RValue asAggregateRValue(CodeGenFunction &CGF) const { - return RValue::getAggregate(getAddress(CGF), isVolatileQualified()); + RValue asAggregateRValue() const { + return RValue::getAggregate(getAddress(), isVolatileQualified()); } }; @@ -607,11 +599,11 @@ public: } static AggValueSlot - forLValue(const LValue &LV, CodeGenFunction &CGF, IsDestructed_t isDestructed, + forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed = IsNotZeroed, IsSanitizerChecked_t isChecked = IsNotSanitizerChecked) { - return forAddr(LV.getAddress(CGF), LV.getQuals(), isDestructed, needsGC, + return forAddr(LV.getAddress(), LV.getQuals(), isDestructed, needsGC, isAliased, mayOverlap, isZeroed, isChecked); } @@ -643,17 +635,17 @@ public: return NeedsGCBarriers_t(ObjCGCFlag); } - llvm::Value *getPointer() const { - return Addr.getPointer(); + llvm::Value *getPointer(QualType PointeeTy, CodeGenFunction &CGF) const; + + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { + return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; } Address getAddress() const { return Addr; } - bool isIgnored() const { - return !Addr.isValid(); - } + bool isIgnored() const { return !Addr.isValid(); } CharUnits getAlignment() const { return Addr.getAlignment(); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp index f8038497d90a..e87226e60297 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp @@ -25,8 +25,11 @@ #include "clang/CodeGen/ModuleBuilder.h" #include "clang/Driver/DriverDiagnostic.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/MultiplexConsumer.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Serialization/ASTWriter.h" #include "llvm/ADT/Hashing.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" @@ -57,10 +60,6 @@ using namespace llvm; #define DEBUG_TYPE "codegenaction" -namespace llvm { -extern cl::opt<bool> ClRelinkBuiltinBitcodePostop; -} - namespace clang { class BackendConsumer; class ClangDiagnosticHandler final : public DiagnosticHandler { @@ -109,56 +108,50 @@ static void reportOptRecordError(Error E, DiagnosticsEngine &Diags, }); } -BackendConsumer::BackendConsumer(BackendAction Action, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, - const HeaderSearchOptions &HeaderSearchOpts, - const PreprocessorOptions &PPOpts, - const CodeGenOptions &CodeGenOpts, - const TargetOptions &TargetOpts, - const LangOptions &LangOpts, - const std::string &InFile, - SmallVector<LinkModule, 4> LinkModules, - std::unique_ptr<raw_pwrite_stream> OS, - LLVMContext &C, - CoverageSourceInfo *CoverageInfo) - : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts), - CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts), - AsmOutStream(std::move(OS)), Context(nullptr), FS(VFS), - LLVMIRGeneration("irgen", "LLVM IR Generation Time"), - LLVMIRGenerationRefCount(0), - Gen(CreateLLVMCodeGen(Diags, InFile, std::move(VFS), HeaderSearchOpts, - PPOpts, CodeGenOpts, C, CoverageInfo)), - LinkModules(std::move(LinkModules)) { - TimerIsEnabled = CodeGenOpts.TimePasses; - llvm::TimePassesIsEnabled = CodeGenOpts.TimePasses; - llvm::TimePassesPerRun = CodeGenOpts.TimePassesPerRun; +BackendConsumer::BackendConsumer( + BackendAction Action, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, + const HeaderSearchOptions &HeaderSearchOpts, + const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts, + const TargetOptions &TargetOpts, const LangOptions &LangOpts, + const std::string &InFile, SmallVector<LinkModule, 4> LinkModules, + std::unique_ptr<raw_pwrite_stream> OS, LLVMContext &C, + CoverageSourceInfo *CoverageInfo) + : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts), + CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts), + AsmOutStream(std::move(OS)), Context(nullptr), FS(VFS), + LLVMIRGeneration("irgen", "LLVM IR Generation Time"), + LLVMIRGenerationRefCount(0), + Gen(CreateLLVMCodeGen(Diags, InFile, std::move(VFS), HeaderSearchOpts, + PPOpts, CodeGenOpts, C, CoverageInfo)), + LinkModules(std::move(LinkModules)) { + TimerIsEnabled = CodeGenOpts.TimePasses; + llvm::TimePassesIsEnabled = CodeGenOpts.TimePasses; + llvm::TimePassesPerRun = CodeGenOpts.TimePassesPerRun; } // This constructor is used in installing an empty BackendConsumer // to use the clang diagnostic handler for IR input files. It avoids // initializing the OS field. -BackendConsumer::BackendConsumer(BackendAction Action, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, - const HeaderSearchOptions &HeaderSearchOpts, - const PreprocessorOptions &PPOpts, - const CodeGenOptions &CodeGenOpts, - const TargetOptions &TargetOpts, - const LangOptions &LangOpts, - llvm::Module *Module, - SmallVector<LinkModule, 4> LinkModules, - LLVMContext &C, - CoverageSourceInfo *CoverageInfo) - : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts), - CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts), - Context(nullptr), FS(VFS), - LLVMIRGeneration("irgen", "LLVM IR Generation Time"), - LLVMIRGenerationRefCount(0), - Gen(CreateLLVMCodeGen(Diags, "", std::move(VFS), HeaderSearchOpts, - PPOpts, CodeGenOpts, C, CoverageInfo)), - LinkModules(std::move(LinkModules)), CurLinkModule(Module) { - TimerIsEnabled = CodeGenOpts.TimePasses; - llvm::TimePassesIsEnabled = CodeGenOpts.TimePasses; - llvm::TimePassesPerRun = CodeGenOpts.TimePassesPerRun; +BackendConsumer::BackendConsumer( + BackendAction Action, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, + const HeaderSearchOptions &HeaderSearchOpts, + const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts, + const TargetOptions &TargetOpts, const LangOptions &LangOpts, + llvm::Module *Module, SmallVector<LinkModule, 4> LinkModules, + LLVMContext &C, CoverageSourceInfo *CoverageInfo) + : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts), + CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts), + Context(nullptr), FS(VFS), + LLVMIRGeneration("irgen", "LLVM IR Generation Time"), + LLVMIRGenerationRefCount(0), + Gen(CreateLLVMCodeGen(Diags, "", std::move(VFS), HeaderSearchOpts, PPOpts, + CodeGenOpts, C, CoverageInfo)), + LinkModules(std::move(LinkModules)), CurLinkModule(Module) { + TimerIsEnabled = CodeGenOpts.TimePasses; + llvm::TimePassesIsEnabled = CodeGenOpts.TimePasses; + llvm::TimePassesPerRun = CodeGenOpts.TimePassesPerRun; } llvm::Module* BackendConsumer::getModule() const { @@ -233,17 +226,11 @@ void BackendConsumer::HandleInterestingDecl(DeclGroupRef D) { HandleTopLevelDecl(D); } -// Links each entry in LinkModules into our module. Returns true on error. -bool BackendConsumer::LinkInModules(llvm::Module *M, bool ShouldLinkFiles) { - +// Links each entry in LinkModules into our module. Returns true on error. +bool BackendConsumer::LinkInModules(llvm::Module *M) { for (auto &LM : LinkModules) { assert(LM.Module && "LinkModule does not actually have a module"); - // If ShouldLinkFiles is not set, skip files added via the - // -mlink-bitcode-files, only linking -mlink-builtin-bitcode - if (!LM.Internalize && !ShouldLinkFiles) - continue; - if (LM.PropagateAttrs) for (Function &F : *LM.Module) { // Skip intrinsics. Keep consistent with how intrinsics are created @@ -257,37 +244,22 @@ bool BackendConsumer::LinkInModules(llvm::Module *M, bool ShouldLinkFiles) { CurLinkModule = LM.Module.get(); bool Err; - auto DoLink = [&](auto &Mod) { - if (LM.Internalize) { - Err = Linker::linkModules( - *M, std::move(Mod), LM.LinkFlags, - [](llvm::Module &M, const llvm::StringSet<> &GVS) { - internalizeModule(M, [&GVS](const llvm::GlobalValue &GV) { - return !GV.hasName() || (GVS.count(GV.getName()) == 0); - }); + if (LM.Internalize) { + Err = Linker::linkModules( + *M, std::move(LM.Module), LM.LinkFlags, + [](llvm::Module &M, const llvm::StringSet<> &GVS) { + internalizeModule(M, [&GVS](const llvm::GlobalValue &GV) { + return !GV.hasName() || (GVS.count(GV.getName()) == 0); }); - } else - Err = Linker::linkModules(*M, std::move(Mod), LM.LinkFlags); - }; + }); + } else + Err = Linker::linkModules(*M, std::move(LM.Module), LM.LinkFlags); - // Create a Clone to move to the linker, which preserves the original - // linking modules, allowing them to be linked again in the future - if (ClRelinkBuiltinBitcodePostop) { - // TODO: If CloneModule() is updated to support cloning of unmaterialized - // modules, we can remove this - if (Error E = CurLinkModule->materializeAll()) - return false; - - std::unique_ptr<llvm::Module> Clone = llvm::CloneModule(*LM.Module); - - DoLink(Clone); - } - // Otherwise we can link (and clean up) the original modules - else { - DoLink(LM.Module); - } + if (Err) + return true; } + LinkModules.clear(); return false; // success } @@ -322,6 +294,9 @@ void BackendConsumer::HandleTranslationUnit(ASTContext &C) { Ctx.setDiagnosticHandler(std::make_unique<ClangDiagnosticHandler>( CodeGenOpts, this)); + Ctx.setDefaultTargetCPU(TargetOpts.CPU); + Ctx.setDefaultTargetFeatures(llvm::join(TargetOpts.Features, ",")); + Expected<std::unique_ptr<llvm::ToolOutputFile>> OptRecordFileOrErr = setupLLVMOptimizationRemarks( Ctx, CodeGenOpts.OptRecordFile, CodeGenOpts.OptRecordPasses, @@ -350,7 +325,7 @@ void BackendConsumer::HandleTranslationUnit(ASTContext &C) { } // Link each LinkModule into our module. - if (LinkInModules(getModule())) + if (!CodeGenOpts.LinkBitcodePostopt && LinkInModules(getModule())) return; for (auto &F : getModule()->functions()) { @@ -401,7 +376,7 @@ void BackendConsumer::CompleteTentativeDefinition(VarDecl *D) { Gen->CompleteTentativeDefinition(D); } -void BackendConsumer::CompleteExternalDeclaration(VarDecl *D) { +void BackendConsumer::CompleteExternalDeclaration(DeclaratorDecl *D) { Gen->CompleteExternalDeclaration(D); } @@ -994,6 +969,12 @@ CodeGenerator *CodeGenAction::getCodeGenerator() const { return BEConsumer->getCodeGenerator(); } +bool CodeGenAction::BeginSourceFileAction(CompilerInstance &CI) { + if (CI.getFrontendOpts().GenReducedBMI) + CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleInterface); + return true; +} + static std::unique_ptr<raw_pwrite_stream> GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) { switch (Action) { @@ -1051,6 +1032,16 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { CI.getPreprocessor().addPPCallbacks(std::move(Callbacks)); } + if (CI.getFrontendOpts().GenReducedBMI && + !CI.getFrontendOpts().ModuleOutputPath.empty()) { + std::vector<std::unique_ptr<ASTConsumer>> Consumers(2); + Consumers[0] = std::make_unique<ReducedBMIGenerator>( + CI.getPreprocessor(), CI.getModuleCache(), + CI.getFrontendOpts().ModuleOutputPath); + Consumers[1] = std::move(Result); + return std::make_unique<MultiplexConsumer>(std::move(Consumers)); + } + return std::move(Result); } @@ -1203,7 +1194,7 @@ void CodeGenAction::ExecuteAction() { std::move(LinkModules), *VMContext, nullptr); // Link in each pending link module. - if (Result.LinkInModules(&*TheModule)) + if (!CodeGenOpts.LinkBitcodePostopt && Result.LinkInModules(&*TheModule)) return; // PR44896: Force DiscardValueNames as false. DiscardValueNames cannot be @@ -1212,6 +1203,9 @@ void CodeGenAction::ExecuteAction() { Ctx.setDiagnosticHandler( std::make_unique<ClangDiagnosticHandler>(CodeGenOpts, &Result)); + Ctx.setDefaultTargetCPU(TargetOpts.CPU); + Ctx.setDefaultTargetFeatures(llvm::join(TargetOpts.Features, ",")); + Expected<std::unique_ptr<llvm::ToolOutputFile>> OptRecordFileOrErr = setupLLVMOptimizationRemarks( Ctx, CodeGenOpts.OptRecordFile, CodeGenOpts.OptRecordPasses, diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp index 2673e4a5cee7..2b2e23f1e5d7 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp @@ -31,6 +31,7 @@ #include "clang/AST/StmtObjC.h" #include "clang/Basic/Builtins.h" #include "clang/Basic/CodeGenOptions.h" +#include "clang/Basic/TargetBuiltins.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/FrontendDiagnostic.h" @@ -52,6 +53,10 @@ using namespace clang; using namespace CodeGen; +namespace llvm { +extern cl::opt<bool> EnableSingleByteCoverage; +} // namespace llvm + /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time /// markers. static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts, @@ -86,6 +91,8 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) CodeGenFunction::~CodeGenFunction() { assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup"); + assert(DeferredDeactivationCleanupStack.empty() && + "missed to deactivate a cleanup"); if (getLangOpts().OpenMP && CurFn) CGM.getOpenMPRuntime().functionFinished(*this); @@ -188,26 +195,47 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() { CGF.Builder.setDefaultConstrainedRounding(OldRounding); } -LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { +static LValue +makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType, + bool MightBeSigned, CodeGenFunction &CGF, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; - CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo); - Address Addr(V, ConvertTypeForMem(T), Alignment); - return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo); + CharUnits Alignment = + CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType); + Address Addr = + MightBeSigned + ? CGF.makeNaturalAddressForPointer(V, T, Alignment, false, nullptr, + nullptr, IsKnownNonNull) + : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull); + return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); +} + +LValue +CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, + KnownNonNull_t IsKnownNonNull) { + return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, + /*MightBeSigned*/ true, *this, + IsKnownNonNull); } -/// Given a value of type T* that may not be to a complete object, -/// construct an l-value with the natural pointee alignment of T. LValue CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { - LValueBaseInfo BaseInfo; - TBAAAccessInfo TBAAInfo; - CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, - /* forPointeeType= */ true); - Address Addr(V, ConvertTypeForMem(T), Align); - return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); + return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, + /*MightBeSigned*/ true, *this); } +LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V, + QualType T) { + return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, + /*MightBeSigned*/ false, *this); +} + +LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, + QualType T) { + return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, + /*MightBeSigned*/ false, *this); +} llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) { return CGM.getTypes().ConvertTypeForMem(T); @@ -217,6 +245,11 @@ llvm::Type *CodeGenFunction::ConvertType(QualType T) { return CGM.getTypes().ConvertType(T); } +llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy, + llvm::Type *LLVMTy) { + return CGM.getTypes().convertTypeForLoadStore(ASTTy, LLVMTy); +} + TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) { type = type.getCanonicalType(); while (true) { @@ -262,6 +295,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) { case Type::Record: case Type::ObjCObject: case Type::ObjCInterface: + case Type::ArrayParameter: return TEK_Aggregate; // We operate on atomic values according to their underlying type. @@ -331,6 +365,16 @@ static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) { void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { assert(BreakContinueStack.empty() && "mismatched push/pop in break/continue stack!"); + assert(LifetimeExtendedCleanupStack.empty() && + "mismatched push/pop of cleanups in EHStack!"); + assert(DeferredDeactivationCleanupStack.empty() && + "mismatched activate/deactivate of cleanups!"); + + if (CGM.shouldEmitConvergenceTokens()) { + ConvergenceTokenStack.pop_back(); + assert(ConvergenceTokenStack.empty() && + "mismatched push/pop in convergence stack!"); + } bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0 && NumSimpleReturnExprs == NumReturnExprs @@ -520,7 +564,8 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { ReturnBlock.getBlock()->eraseFromParent(); } if (ReturnValue.isValid()) { - auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer()); + auto *RetAlloca = + dyn_cast<llvm::AllocaInst>(ReturnValue.emitRawPointer(*this)); if (RetAlloca && RetAlloca->use_empty()) { RetAlloca->eraseFromParent(); ReturnValue = Address::invalid(); @@ -790,6 +835,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, Fn->addFnAttr(llvm::Attribute::SanitizeMemTag); if (SanOpts.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); + if (SanOpts.has(SanitizerKind::NumericalStability)) + Fn->addFnAttr(llvm::Attribute::SanitizeNumericalStability); if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory)) Fn->addFnAttr(llvm::Attribute::SanitizeMemory); } @@ -806,7 +853,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time. if (SanOpts.has(SanitizerKind::Thread)) { if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) { - IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0); + const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0); if (OMD->getMethodFamily() == OMF_dealloc || OMD->getMethodFamily() == OMF_initialize || (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) { @@ -831,6 +878,17 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass) SanOpts.Mask &= ~SanitizerKind::Null; + // Add pointer authentication attributes. + const CodeGenOptions &CodeGenOpts = CGM.getCodeGenOpts(); + if (CodeGenOpts.PointerAuth.ReturnAddresses) + Fn->addFnAttr("ptrauth-returns"); + if (CodeGenOpts.PointerAuth.FunctionPointers) + Fn->addFnAttr("ptrauth-calls"); + if (CodeGenOpts.PointerAuth.AuthTraps) + Fn->addFnAttr("ptrauth-auth-traps"); + if (CodeGenOpts.PointerAuth.IndirectGotos) + Fn->addFnAttr("ptrauth-indirect-gotos"); + // Apply xray attributes to the function (as a string, for now) bool AlwaysXRayAttr = false; if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) { @@ -937,6 +995,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, if (D && D->hasAttr<NoProfileFunctionAttr>()) Fn->addFnAttr(llvm::Attribute::NoProfile); + if (D && D->hasAttr<HybridPatchableAttr>()) + Fn->addFnAttr(llvm::Attribute::HybridPatchable); + if (D) { // Function attributes take precedence over command line flags. if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) { @@ -957,6 +1018,11 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, EmitKernelMetadata(FD, Fn); } + if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) { + Fn->setMetadata("clspv_libclc_builtin", + llvm::MDNode::get(getLLVMContext(), {})); + } + // If we are checking function types, emit a function type signature as // prologue data. if (FD && SanOpts.has(SanitizerKind::Function)) { @@ -974,7 +1040,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // return value. Initialize the flag to 'true' and refine it in EmitParmDecl. if (SanOpts.has(SanitizerKind::NullabilityReturn)) { auto Nullability = FnRetTy->getNullability(); - if (Nullability && *Nullability == NullabilityKind::NonNull) { + if (Nullability && *Nullability == NullabilityKind::NonNull && + !FnRetTy->isRecordType()) { if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>())) RetValNullabilityPrecondition = @@ -1117,13 +1184,14 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = - Address(&*AI, ConvertType(RetTy), - CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull); + ReturnValue = makeNaturalAddressForPointer( + &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false, + nullptr, nullptr, KnownNonNull); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { - ReturnValuePointer = CreateDefaultAlignTempAlloca( - ReturnValue.getPointer()->getType(), "result.ptr"); - Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer); + ReturnValuePointer = + CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr"); + Builder.CreateStore(ReturnValue.emitRawPointer(*this), + ReturnValuePointer); } } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { @@ -1184,8 +1252,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // or contains the address of the enclosing object). LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - // If the enclosing object was captured by value, just use its address. - CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); + // If the enclosing object was captured by value, just use its + // address. Sign this pointer. + CXXThisValue = ThisFieldLValue.getPointer(*this); } else { // Load the lvalue pointed to by the field, since '*this' was captured // by reference. @@ -1252,6 +1321,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, if (CurFuncDecl) if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>()) LargestVectorWidth = VecWidth->getVectorWidth(); + + if (CGM.shouldEmitConvergenceTokens()) + ConvergenceTokenStack.push_back(getOrEmitConvergenceEntryToken(CurFn)); } void CodeGenFunction::EmitFunctionBody(const Stmt *Body) { @@ -1270,7 +1342,10 @@ void CodeGenFunction::EmitFunctionBody(const Stmt *Body) { void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S) { llvm::BasicBlock *SkipCountBB = nullptr; - if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) { + // Do not skip over the instrumentation when single byte coverage mode is + // enabled. + if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr() && + !llvm::EnableSingleByteCoverage) { // When instrumenting for profiling, the fallthrough to certain // statements needs to skip over the instrumentation code so that we // get an accurate count. @@ -1353,6 +1428,8 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, FunctionArgList Args; QualType ResTy = BuildFunctionArgList(GD, Args); + CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, FD); + if (FD->isInlineBuiltinDeclaration()) { // When generating code for a builtin with an inline declaration, use a // mangled name to hold the actual body, while keeping an external @@ -1441,6 +1518,8 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, // Ensure that the function adheres to the forward progress guarantee, which // is required by certain optimizations. + // In C++11 and up, the attribute will be removed if the body contains a + // trivial empty loop. if (checkIfFunctionMustProgress()) CurFn->addFnAttr(llvm::Attribute::MustProgress); @@ -2002,8 +2081,9 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); Address begin = dest.withElementType(CGF.Int8Ty); - llvm::Value *end = Builder.CreateInBoundsGEP( - begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end"); + llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(), + begin.emitRawPointer(CGF), + sizeInChars, "vla.end"); llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock(); llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop"); @@ -2014,7 +2094,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(loopBB); llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur"); - cur->addIncoming(begin.getPointer(), originBB); + cur->addIncoming(begin.emitRawPointer(CGF), originBB); CharUnits curAlign = dest.getAlignment().alignmentOfArrayElement(baseSize); @@ -2179,8 +2259,8 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, dyn_cast<llvm::ArrayType>(addr.getElementType()); while (llvmArrayType) { assert(isa<ConstantArrayType>(arrayType)); - assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue() - == llvmArrayType->getNumElements()); + assert(cast<ConstantArrayType>(arrayType)->getZExtSize() == + llvmArrayType->getNumElements()); gepIndices.push_back(zero); countFromCLAs *= llvmArrayType->getNumElements(); @@ -2198,8 +2278,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, // as some other type (probably a packed struct). Compute the array // size, and just emit the 'begin' expression as a bitcast. while (arrayType) { - countFromCLAs *= - cast<ConstantArrayType>(arrayType)->getSize().getZExtValue(); + countFromCLAs *= cast<ConstantArrayType>(arrayType)->getZExtSize(); eltType = arrayType->getElementType(); arrayType = getContext().getAsArrayType(eltType); } @@ -2208,10 +2287,10 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, addr = addr.withElementType(baseType); } else { // Create the actual GEP. - addr = Address(Builder.CreateInBoundsGEP( - addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"), - ConvertTypeForMem(eltType), - addr.getAlignment()); + addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(), + addr.emitRawPointer(*this), + gepIndices, "array.begin"), + ConvertTypeForMem(eltType), addr.getAlignment()); } baseType = eltType; @@ -2339,6 +2418,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { type = cast<MemberPointerType>(ty)->getPointeeType(); break; + case Type::ArrayParameter: case Type::ConstantArray: case Type::IncompleteArray: // Losing element qualification here is fine. @@ -2399,6 +2479,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { case Type::BTFTagAttributed: case Type::SubstTemplateTypeParm: case Type::MacroQualified: + case Type::CountAttributed: // Keep walking after single level desugaring. type = type.getSingleStepDesugaredType(getContext()); break; @@ -2407,6 +2488,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { case Type::Decltype: case Type::Auto: case Type::DeducedTemplateSpecialization: + case Type::PackIndexing: // Stop walking: nothing to do. return; @@ -2429,11 +2511,11 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { Address CodeGenFunction::EmitVAListRef(const Expr* E) { if (getContext().getBuiltinVaListType()->isArrayType()) return EmitPointerWithAlignment(E); - return EmitLValue(E).getAddress(*this); + return EmitLValue(E).getAddress(); } Address CodeGenFunction::EmitMSVAListRef(const Expr *E) { - return EmitLValue(E).getAddress(*this); + return EmitLValue(E).getAddress(); } void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E, @@ -2550,7 +2632,7 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, Address Addr) { assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute"); - llvm::Value *V = Addr.getPointer(); + llvm::Value *V = Addr.emitRawPointer(*this); llvm::Type *VTy = V->getType(); auto *PTy = dyn_cast<llvm::PointerType>(VTy); unsigned AS = PTy ? PTy->getAddressSpace() : 0; @@ -2586,7 +2668,6 @@ CodeGenFunction::SanitizerScope::~SanitizerScope() { void CodeGenFunction::InsertHelper(llvm::Instruction *I, const llvm::Twine &Name, - llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const { LoopStack.InsertHelper(I); if (IsSanitizerScope) @@ -2594,17 +2675,35 @@ void CodeGenFunction::InsertHelper(llvm::Instruction *I, } void CGBuilderInserter::InsertHelper( - llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB, + llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock::iterator InsertPt) const { - llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt); + llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt); if (CGF) - CGF->InsertHelper(I, Name, BB, InsertPt); + CGF->InsertHelper(I, Name, InsertPt); } // Emits an error if we don't have a valid set of target features for the // called function. void CodeGenFunction::checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl) { + // SemaChecking cannot handle below x86 builtins because they have different + // parameter ranges with different TargetAttribute of caller. + if (CGM.getContext().getTargetInfo().getTriple().isX86()) { + unsigned BuiltinID = TargetDecl->getBuiltinID(); + if (BuiltinID == X86::BI__builtin_ia32_cmpps || + BuiltinID == X86::BI__builtin_ia32_cmpss || + BuiltinID == X86::BI__builtin_ia32_cmppd || + BuiltinID == X86::BI__builtin_ia32_cmpsd) { + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl); + llvm::StringMap<bool> TargetFetureMap; + CGM.getContext().getFunctionFeatureMap(TargetFetureMap, FD); + llvm::APSInt Result = + *(E->getArg(2)->getIntegerConstantExpr(CGM.getContext())); + if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup("avx")) + CGM.getDiags().Report(E->getBeginLoc(), diag::err_builtin_needs_feature) + << TargetDecl->getDeclName() << "avx"; + } + } return checkTargetFeatures(E->getBeginLoc(), TargetDecl); } @@ -2707,11 +2806,8 @@ void CodeGenFunction::EmitKCFIOperandBundle( llvm::Value *CodeGenFunction::FormAArch64ResolverCondition( const MultiVersionResolverOption &RO) { llvm::SmallVector<StringRef, 8> CondFeatures; - for (const StringRef &Feature : RO.Conditions.Features) { - // Form condition for features which are not yet enabled in target - if (!getContext().getTargetInfo().hasFeature(Feature)) - CondFeatures.push_back(Feature); - } + for (const StringRef &Feature : RO.Conditions.Features) + CondFeatures.push_back(Feature); if (!CondFeatures.empty()) { return EmitAArch64CpuSupports(CondFeatures); } @@ -2879,7 +2975,7 @@ void CodeGenFunction::emitAlignmentAssumptionCheck( SourceLocation SecondaryLoc, llvm::Value *Alignment, llvm::Value *OffsetValue, llvm::Value *TheCheck, llvm::Instruction *Assumption) { - assert(Assumption && isa<llvm::CallInst>(Assumption) && + assert(isa_and_nonnull<llvm::CallInst>(Assumption) && cast<llvm::CallInst>(Assumption)->getCalledOperand() == llvm::Intrinsic::getDeclaration( Builder.GetInsertBlock()->getParent()->getParent(), @@ -2969,3 +3065,82 @@ llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec, return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name); } + +void CodeGenFunction::EmitPointerAuthOperandBundle( + const CGPointerAuthInfo &PointerAuth, + SmallVectorImpl<llvm::OperandBundleDef> &Bundles) { + if (!PointerAuth.isSigned()) + return; + + auto *Key = Builder.getInt32(PointerAuth.getKey()); + + llvm::Value *Discriminator = PointerAuth.getDiscriminator(); + if (!Discriminator) + Discriminator = Builder.getSize(0); + + llvm::Value *Args[] = {Key, Discriminator}; + Bundles.emplace_back("ptrauth", Args); +} + +static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF, + const CGPointerAuthInfo &PointerAuth, + llvm::Value *Pointer, + unsigned IntrinsicID) { + if (!PointerAuth) + return Pointer; + + auto Key = CGF.Builder.getInt32(PointerAuth.getKey()); + + llvm::Value *Discriminator = PointerAuth.getDiscriminator(); + if (!Discriminator) { + Discriminator = CGF.Builder.getSize(0); + } + + // Convert the pointer to intptr_t before signing it. + auto OrigType = Pointer->getType(); + Pointer = CGF.Builder.CreatePtrToInt(Pointer, CGF.IntPtrTy); + + // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator) + auto Intrinsic = CGF.CGM.getIntrinsic(IntrinsicID); + Pointer = CGF.EmitRuntimeCall(Intrinsic, {Pointer, Key, Discriminator}); + + // Convert back to the original type. + Pointer = CGF.Builder.CreateIntToPtr(Pointer, OrigType); + return Pointer; +} + +llvm::Value * +CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth, + llvm::Value *Pointer) { + if (!PointerAuth.shouldSign()) + return Pointer; + return EmitPointerAuthCommon(*this, PointerAuth, Pointer, + llvm::Intrinsic::ptrauth_sign); +} + +static llvm::Value *EmitStrip(CodeGenFunction &CGF, + const CGPointerAuthInfo &PointerAuth, + llvm::Value *Pointer) { + auto StripIntrinsic = CGF.CGM.getIntrinsic(llvm::Intrinsic::ptrauth_strip); + + auto Key = CGF.Builder.getInt32(PointerAuth.getKey()); + // Convert the pointer to intptr_t before signing it. + auto OrigType = Pointer->getType(); + Pointer = CGF.EmitRuntimeCall( + StripIntrinsic, {CGF.Builder.CreatePtrToInt(Pointer, CGF.IntPtrTy), Key}); + return CGF.Builder.CreateIntToPtr(Pointer, OrigType); +} + +llvm::Value * +CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth, + llvm::Value *Pointer) { + if (PointerAuth.shouldStrip()) { + return EmitStrip(*this, PointerAuth, Pointer); + } + if (!PointerAuth.shouldAuth()) { + return Pointer; + } + + return EmitPointerAuthCommon(*this, PointerAuth, Pointer, + llvm::Intrinsic::ptrauth_auth); +} diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h index 143ad64e8816..60e6841e1b3d 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.h @@ -26,6 +26,7 @@ #include "clang/AST/ExprCXX.h" #include "clang/AST/ExprObjC.h" #include "clang/AST/ExprOpenMP.h" +#include "clang/AST/StmtOpenACC.h" #include "clang/AST/StmtOpenMP.h" #include "clang/AST/Type.h" #include "clang/Basic/ABI.h" @@ -38,6 +39,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Debug.h" #include "llvm/Transforms/Utils/SanitizerStats.h" @@ -135,7 +137,8 @@ enum TypeEvaluationKind { SANITIZER_CHECK(SubOverflow, sub_overflow, 0) \ SANITIZER_CHECK(TypeMismatch, type_mismatch, 1) \ SANITIZER_CHECK(AlignmentAssumption, alignment_assumption, 0) \ - SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0) + SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0) \ + SANITIZER_CHECK(BoundsSafety, bounds_safety, 0) enum SanitizerHandler { #define SANITIZER_CHECK(Enum, Name, Version) Enum, @@ -150,6 +153,9 @@ struct DominatingLLVMValue { /// Answer whether the given value needs extra work to be saved. static bool needsSaving(llvm::Value *value) { + if (!value) + return false; + // If it's not an instruction, we don't need to save. if (!isa<llvm::Instruction>(value)) return false; @@ -176,21 +182,28 @@ template <> struct DominatingValue<Address> { typedef Address type; struct saved_type { - DominatingLLVMValue::saved_type SavedValue; + DominatingLLVMValue::saved_type BasePtr; llvm::Type *ElementType; CharUnits Alignment; + DominatingLLVMValue::saved_type Offset; + llvm::PointerType *EffectiveType; }; static bool needsSaving(type value) { - return DominatingLLVMValue::needsSaving(value.getPointer()); + if (DominatingLLVMValue::needsSaving(value.getBasePointer()) || + DominatingLLVMValue::needsSaving(value.getOffset())) + return true; + return false; } static saved_type save(CodeGenFunction &CGF, type value) { - return { DominatingLLVMValue::save(CGF, value.getPointer()), - value.getElementType(), value.getAlignment() }; + return {DominatingLLVMValue::save(CGF, value.getBasePointer()), + value.getElementType(), value.getAlignment(), + DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()}; } static type restore(CodeGenFunction &CGF, saved_type value) { - return Address(DominatingLLVMValue::restore(CGF, value.SavedValue), - value.ElementType, value.Alignment); + return Address(DominatingLLVMValue::restore(CGF, value.BasePtr), + value.ElementType, value.Alignment, CGPointerAuthInfo(), + DominatingLLVMValue::restore(CGF, value.Offset)); } }; @@ -200,13 +213,24 @@ template <> struct DominatingValue<RValue> { class saved_type { enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral, AggregateAddress, ComplexAddress }; - - llvm::Value *Value; - llvm::Type *ElementType; + union { + struct { + DominatingLLVMValue::saved_type first, second; + } Vals; + DominatingValue<Address>::saved_type AggregateAddr; + }; + LLVM_PREFERRED_TYPE(Kind) unsigned K : 3; - unsigned Align : 29; - saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0) - : Value(v), ElementType(e), K(k), Align(a) {} + + saved_type(DominatingLLVMValue::saved_type Val1, unsigned K) + : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {} + + saved_type(DominatingLLVMValue::saved_type Val1, + DominatingLLVMValue::saved_type Val2) + : Vals{Val1, Val2}, K(ComplexAddress) {} + + saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K) + : AggregateAddr(AggregateAddr), K(K) {} public: static bool needsSaving(RValue value); @@ -290,6 +314,9 @@ public: /// Stack to track the Logical Operator recursion nest for MC/DC. SmallVector<const BinaryOperator *, 16> MCDCLogOpStack; + /// Stack to track the controlled convergence tokens. + SmallVector<llvm::IntrinsicInst *, 4> ConvergenceTokenStack; + /// Number of nested loop to be consumed by the last surrounding /// loop-associated directive. int ExpectedOMPLoopDepth = 0; @@ -316,7 +343,6 @@ public: /// CGBuilder insert helper. This function is called after an /// instruction is created using Builder. void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name, - llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const; /// CurFuncDecl - Holds the Decl for the current outermost @@ -350,6 +376,25 @@ public: return isCoroutine() && CurCoro.InSuspendBlock; } + // Holds FramePtr for await_suspend wrapper generation, + // so that __builtin_coro_frame call can be lowered + // directly to value of its second argument + struct AwaitSuspendWrapperInfo { + llvm::Value *FramePtr = nullptr; + }; + AwaitSuspendWrapperInfo CurAwaitSuspendWrapper; + + // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics. + // It encapsulates SuspendExpr in a function, to separate it's body + // from the main coroutine to avoid miscompilations. Intrinisic + // is lowered to this function call in CoroSplit pass + // Function signature is: + // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl) + // where type is one of (void, i1, ptr) + llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName, + Twine const &SuspendPointName, + CoroutineSuspendExpr const &S); + /// CurGD - The GlobalDecl for the current function being compiled. GlobalDecl CurGD; @@ -592,28 +637,7 @@ public: /// Returns true if a loop must make progress, which means the mustprogress /// attribute can be added. \p HasConstantCond indicates whether the branch /// condition is a known constant. - bool checkIfLoopMustProgress(bool HasConstantCond) { - if (CGM.getCodeGenOpts().getFiniteLoops() == - CodeGenOptions::FiniteLoopsKind::Always) - return true; - if (CGM.getCodeGenOpts().getFiniteLoops() == - CodeGenOptions::FiniteLoopsKind::Never) - return false; - - // If the containing function must make progress, loops also must make - // progress (as in C++11 and later). - if (checkIfFunctionMustProgress()) - return true; - - // Now apply rules for plain C (see 6.8.5.6 in C11). - // Loops with constant conditions do not have to make progress in any C - // version. - if (HasConstantCond) - return false; - - // Loops with non-constant conditions must make progress in C11 and later. - return getLangOpts().C11; - } + bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody); const CodeGen::CGBlockInfo *BlockInfo = nullptr; llvm::Value *BlockPointer = nullptr; @@ -627,6 +651,51 @@ public: EHScopeStack EHStack; llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack; + + // A stack of cleanups which were added to EHStack but have to be deactivated + // later before being popped or emitted. These are usually deactivated on + // exiting a `CleanupDeactivationScope` scope. For instance, after a + // full-expr. + // + // These are specially useful for correctly emitting cleanups while + // encountering branches out of expression (through stmt-expr or coroutine + // suspensions). + struct DeferredDeactivateCleanup { + EHScopeStack::stable_iterator Cleanup; + llvm::Instruction *DominatingIP; + }; + llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack; + + // Enters a new scope for capturing cleanups which are deferred to be + // deactivated, all of which will be deactivated once the scope is exited. + struct CleanupDeactivationScope { + CodeGenFunction &CGF; + size_t OldDeactivateCleanupStackSize; + bool Deactivated; + CleanupDeactivationScope(CodeGenFunction &CGF) + : CGF(CGF), OldDeactivateCleanupStackSize( + CGF.DeferredDeactivationCleanupStack.size()), + Deactivated(false) {} + + void ForceDeactivate() { + assert(!Deactivated && "Deactivating already deactivated scope"); + auto &Stack = CGF.DeferredDeactivationCleanupStack; + for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) { + CGF.DeactivateCleanupBlock(Stack[I - 1].Cleanup, + Stack[I - 1].DominatingIP); + Stack[I - 1].DominatingIP->eraseFromParent(); + } + Stack.resize(OldDeactivateCleanupStackSize); + Deactivated = true; + } + + ~CleanupDeactivationScope() { + if (Deactivated) + return; + ForceDeactivate(); + } + }; + llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack; llvm::Instruction *CurrentFuncletPad = nullptr; @@ -638,7 +707,7 @@ public: llvm::Value *Size; public: - CallLifetimeEnd(Address addr, llvm::Value *size) + CallLifetimeEnd(RawAddress addr, llvm::Value *size) : Addr(addr.getPointer()), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -650,9 +719,11 @@ public: struct LifetimeExtendedCleanupHeader { /// The size of the following cleanup object. unsigned Size; - /// The kind of cleanup to push: a value from the CleanupKind enumeration. + /// The kind of cleanup to push. + LLVM_PREFERRED_TYPE(CleanupKind) unsigned Kind : 31; /// Whether this is a conditional cleanup. + LLVM_PREFERRED_TYPE(bool) unsigned IsConditional : 1; size_t getSize() const { return Size; } @@ -661,7 +732,7 @@ public: }; /// i32s containing the indexes of the cleanup destinations. - Address NormalCleanupDest = Address::invalid(); + RawAddress NormalCleanupDest = RawAddress::invalid(); unsigned NextCleanupDestIndex = 1; @@ -796,10 +867,10 @@ public: template <class T, class... As> void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) { if (!isInConditionalBranch()) - return pushCleanupAfterFullExprWithActiveFlag<T>(Kind, Address::invalid(), - A...); + return pushCleanupAfterFullExprWithActiveFlag<T>( + Kind, RawAddress::invalid(), A...); - Address ActiveFlag = createCleanupActiveFlag(); + RawAddress ActiveFlag = createCleanupActiveFlag(); assert(!DominatingValue<Address>::needsSaving(ActiveFlag) && "cleanup active flag should never need saving"); @@ -812,7 +883,7 @@ public: template <class T, class... As> void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind, - Address ActiveFlag, As... A) { + RawAddress ActiveFlag, As... A) { LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind, ActiveFlag.isValid()}; @@ -827,7 +898,20 @@ public: new (Buffer) LifetimeExtendedCleanupHeader(Header); new (Buffer + sizeof(Header)) T(A...); if (Header.IsConditional) - new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag); + new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag); + } + + // Push a cleanup onto EHStack and deactivate it later. It is usually + // deactivated when exiting a `CleanupDeactivationScope` (for example: after a + // full expression). + template <class T, class... As> + void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) { + // Placeholder dominating IP for this cleanup. + llvm::Instruction *DominatingIP = + Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy)); + EHStack.pushCleanup<T>(Kind, A...); + DeferredDeactivationCleanupStack.push_back( + {EHStack.stable_begin(), DominatingIP}); } /// Set up the last cleanup that was pushed as a conditional @@ -836,8 +920,8 @@ public: initFullExprCleanupWithFlag(createCleanupActiveFlag()); } - void initFullExprCleanupWithFlag(Address ActiveFlag); - Address createCleanupActiveFlag(); + void initFullExprCleanupWithFlag(RawAddress ActiveFlag); + RawAddress createCleanupActiveFlag(); /// PushDestructorCleanup - Push a cleanup to call the /// complete-object destructor of an object of the given type at the @@ -853,7 +937,8 @@ public: /// PopCleanupBlock - Will pop the cleanup entry on the stack and /// process all branch fixups. - void PopCleanupBlock(bool FallThroughIsBranchThrough = false); + void PopCleanupBlock(bool FallThroughIsBranchThrough = false, + bool ForDeactivation = false); /// DeactivateCleanupBlock - Deactivates the given cleanup block. /// The block cannot be reactivated. Pops it if it's the top of the @@ -881,6 +966,7 @@ public: class RunCleanupsScope { EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth; size_t LifetimeExtendedCleanupStackSize; + CleanupDeactivationScope DeactivateCleanups; bool OldDidCallStackSave; protected: bool PerformCleanup; @@ -895,8 +981,7 @@ public: public: /// Enter a new cleanup scope. explicit RunCleanupsScope(CodeGenFunction &CGF) - : PerformCleanup(true), CGF(CGF) - { + : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) { CleanupStackDepth = CGF.EHStack.stable_begin(); LifetimeExtendedCleanupStackSize = CGF.LifetimeExtendedCleanupStack.size(); @@ -926,6 +1011,7 @@ public: void ForceCleanup(std::initializer_list<llvm::Value**> ValuesToReload = {}) { assert(PerformCleanup && "Already forced cleanup"); CGF.DidCallStackSave = OldDidCallStackSave; + DeactivateCleanups.ForceDeactivate(); CGF.PopCleanupBlocks(CleanupStackDepth, LifetimeExtendedCleanupStackSize, ValuesToReload); PerformCleanup = false; @@ -1025,7 +1111,7 @@ public: QualType VarTy = LocalVD->getType(); if (VarTy->isReferenceType()) { Address Temp = CGF.CreateMemTemp(VarTy); - CGF.Builder.CreateStore(TempAddr.getPointer(), Temp); + CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp); TempAddr = Temp; } SavedTempAddresses.try_emplace(LocalVD, TempAddr); @@ -1220,10 +1306,12 @@ public: /// one branch or the other of a conditional expression. bool isInConditionalBranch() const { return OutermostConditional != nullptr; } - void setBeforeOutermostConditional(llvm::Value *value, Address addr) { + void setBeforeOutermostConditional(llvm::Value *value, Address addr, + CodeGenFunction &CGF) { assert(isInConditionalBranch()); llvm::BasicBlock *block = OutermostConditional->getStartingBlock(); - auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back()); + auto store = + new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back()); store->setAlignment(addr.getAlignment().getAsAlign()); } @@ -1540,8 +1628,10 @@ public: void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) { if (CGM.getCodeGenOpts().hasProfileClangInstr() && !CurFn->hasFnAttribute(llvm::Attribute::NoProfile) && - !CurFn->hasFnAttribute(llvm::Attribute::SkipProfile)) - PGO.emitCounterIncrement(Builder, S, StepV); + !CurFn->hasFnAttribute(llvm::Attribute::SkipProfile)) { + auto AL = ApplyDebugLocation::CreateArtificial(*this); + PGO.emitCounterSetOrIncrement(Builder, S, StepV); + } PGO.setCurrentStmt(S); } @@ -1578,7 +1668,7 @@ public: /// If \p StepV is null, the default increment is 1. void maybeUpdateMCDCTestVectorBitmap(const Expr *E) { if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) { - PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr); + PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this); PGO.setCurrentStmt(E); } } @@ -1586,7 +1676,7 @@ public: /// Update the MCDC temp value with the condition's evaluated result. void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) { if (isMCDCCoverageEnabled()) { - PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val); + PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this); PGO.setCurrentStmt(E); } } @@ -1681,7 +1771,7 @@ public: : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue), OldCXXThisAlignment(CGF.CXXThisAlignment), SourceLocScope(E, CGF.CurSourceLocExprScope) { - CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer(); + CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer(); CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment(); } ~CXXDefaultInitExprScope() { @@ -2067,7 +2157,7 @@ public: llvm::Value *getExceptionFromSlot(); llvm::Value *getSelectorFromSlot(); - Address getNormalCleanupDestSlot(); + RawAddress getNormalCleanupDestSlot(); llvm::BasicBlock *getUnreachableBlock() { if (!UnreachableBlock) { @@ -2113,6 +2203,11 @@ public: Address addr, QualType type); void pushDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray); + void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind, + Address addr, QualType type); + void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr, + QualType type, Destroyer *destroyer, + bool useEHCleanupForArray); void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray); @@ -2358,10 +2453,20 @@ public: void InitializeVTablePointers(const CXXRecordDecl *ClassDecl); + // VTableTrapMode - whether we guarantee that loading the + // vtable is guaranteed to trap on authentication failure, + // even if the resulting vtable pointer is unused. + enum class VTableAuthMode { + Authenticate, + MustTrap, + UnsafeUbsanStrip // Should only be used for Vptr UBSan check + }; /// GetVTablePtr - Return the Value of the vtable pointer member pointed /// to by This. - llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy, - const CXXRecordDecl *VTableClass); + llvm::Value * + GetVTablePtr(Address This, llvm::Type *VTableTy, + const CXXRecordDecl *VTableClass, + VTableAuthMode AuthMode = VTableAuthMode::Authenticate); enum CFITypeCheckKind { CFITCK_VCall, @@ -2471,6 +2576,8 @@ public: llvm::Type *ConvertTypeForMem(QualType T); llvm::Type *ConvertType(QualType T); + llvm::Type *convertTypeForLoadStore(QualType ASTTy, + llvm::Type *LLVMTy = nullptr); llvm::Type *ConvertType(const TypeDecl *T) { return ConvertType(getContext().getTypeDeclType(T)); } @@ -2556,10 +2663,41 @@ public: // Helpers //===--------------------------------------------------------------------===// + Address mergeAddressesInConditionalExpr(Address LHS, Address RHS, + llvm::BasicBlock *LHSBlock, + llvm::BasicBlock *RHSBlock, + llvm::BasicBlock *MergeBlock, + QualType MergedType) { + Builder.SetInsertPoint(MergeBlock); + llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond"); + PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock); + PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock); + LHS.replaceBasePointer(PtrPhi); + LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment())); + return LHS; + } + + /// Construct an address with the natural alignment of T. If a pointer to T + /// is expected to be signed, the pointer passed to this function must have + /// been signed, and the returned Address will have the pointer authentication + /// information needed to authenticate the signed pointer. + Address makeNaturalAddressForPointer( + llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(), + bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr, + TBAAAccessInfo *TBAAInfo = nullptr, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) { + if (Alignment.isZero()) + Alignment = + CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType); + return Address(Ptr, ConvertTypeForMem(T), Alignment, + CGM.getPointerAuthInfoForPointeeType(T), /*Offset=*/nullptr, + IsKnownNonNull); + } + LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source = AlignmentSource::Type) { - return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), - CGM.getTBAAAccessInfo(T)); + return MakeAddrLValue(Addr, T, LValueBaseInfo(Source), + CGM.getTBAAAccessInfo(T)); } LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo, @@ -2569,6 +2707,14 @@ public: LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source = AlignmentSource::Type) { + return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T, + LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); + } + + /// Same as MakeAddrLValue above except that the pointer is known to be + /// unsigned. + LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, + AlignmentSource Source = AlignmentSource::Type) { Address Addr(V, ConvertTypeForMem(T), Alignment); return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); @@ -2581,8 +2727,19 @@ public: TBAAAccessInfo()); } + /// Given a value of type T* that may not be to a complete object, construct + /// an l-value with the natural pointee alignment of T. LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T); - LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T); + + LValue + MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull); + + /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known + /// to be unsigned. + LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T); + + LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T); Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo = nullptr, @@ -2604,6 +2761,33 @@ public: TBAAAccessInfo *TBAAInfo = nullptr); LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy); +private: + struct AllocaTracker { + void Add(llvm::AllocaInst *I) { Allocas.push_back(I); } + llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); } + + private: + llvm::SmallVector<llvm::AllocaInst *> Allocas; + }; + AllocaTracker *Allocas = nullptr; + +public: + // Captures all the allocas created during the scope of its RAII object. + struct AllocaTrackerRAII { + AllocaTrackerRAII(CodeGenFunction &CGF) + : CGF(CGF), OldTracker(CGF.Allocas) { + CGF.Allocas = &Tracker; + } + ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; } + + llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); } + + private: + CodeGenFunction &CGF; + AllocaTracker *OldTracker; + AllocaTracker Tracker; + }; + /// CreateTempAlloca - This creates an alloca and inserts it into the entry /// block if \p ArraySize is nullptr, otherwise inserts it at the current /// insertion point of the builder. The caller is responsible for setting an @@ -2632,13 +2816,13 @@ public: /// more efficient if the caller knows that the address will not be exposed. llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp", llvm::Value *ArraySize = nullptr); - Address CreateTempAlloca(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr, - Address *Alloca = nullptr); - Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr); + RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr, + RawAddress *Alloca = nullptr); + RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr); /// CreateDefaultAlignedTempAlloca - This creates an alloca with the /// default ABI alignment of the given LLVM type. @@ -2650,8 +2834,8 @@ public: /// not hand this address off to arbitrary IRGen routines, and especially /// do not pass it as an argument to a function that might expect a /// properly ABI-aligned value. - Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name = "tmp"); + RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name = "tmp"); /// CreateIRTemp - Create a temporary IR object of the given type, with /// appropriate alignment. This routine should only be used when an temporary @@ -2661,38 +2845,52 @@ public: /// /// That is, this is exactly equivalent to CreateMemTemp, but calling /// ConvertType instead of ConvertTypeForMem. - Address CreateIRTemp(QualType T, const Twine &Name = "tmp"); + RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp"); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen and cast it to the default address space. Returns /// the original alloca instruction by \p Alloca if it is not nullptr. - Address CreateMemTemp(QualType T, const Twine &Name = "tmp", - Address *Alloca = nullptr); - Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp", - Address *Alloca = nullptr); + RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp", + RawAddress *Alloca = nullptr); + RawAddress CreateMemTemp(QualType T, CharUnits Align, + const Twine &Name = "tmp", + RawAddress *Alloca = nullptr); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen without casting it to the default address space. - Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); - Address CreateMemTempWithoutCast(QualType T, CharUnits Align, - const Twine &Name = "tmp"); + RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); + RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align, + const Twine &Name = "tmp"); /// CreateAggTemp - Create a temporary memory object for the given /// aggregate type. AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp", - Address *Alloca = nullptr) { - return AggValueSlot::forAddr(CreateMemTemp(T, Name, Alloca), - T.getQualifiers(), - AggValueSlot::IsNotDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased, - AggValueSlot::DoesNotOverlap); + RawAddress *Alloca = nullptr) { + return AggValueSlot::forAddr( + CreateMemTemp(T, Name, Alloca), T.getQualifiers(), + AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap); } /// EvaluateExprAsBool - Perform the usual unary conversions on the specified /// expression and compare the result against zero, returning an Int1Ty value. llvm::Value *EvaluateExprAsBool(const Expr *E); + /// Retrieve the implicit cast expression of the rhs in a binary operator + /// expression by passing pointers to Value and QualType + /// This is used for implicit bitfield conversion checks, which + /// must compare with the value before potential truncation. + llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E, + llvm::Value **Previous, + QualType *SrcType); + + /// Emit a check that an [implicit] conversion of a bitfield. It is not UB, + /// so we use the value after conversion. + void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType, + llvm::Value *Dst, QualType DstType, + const CGBitFieldInfo &Info, + SourceLocation Loc); + /// EmitIgnoredExpr - Emit an expression in a context which ignores the result. void EmitIgnoredExpr(const Expr *E); @@ -2830,7 +3028,8 @@ public: /// \returns A pointer to the argument. // FIXME: We should be able to get rid of this method and use the va_arg // instruction in LLVM instead once it works well enough. - Address EmitVAArg(VAArgExpr *VE, Address &VAListAddr); + RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr, + AggValueSlot Slot = AggValueSlot::ignored()); /// emitArrayLength - Compute the length of an array, even if it's a /// VLA, and drill down to the base element type. @@ -3060,6 +3259,25 @@ public: /// calls to EmitTypeCheck can be skipped. bool sanitizePerformTypeCheck() const; + void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, + QualType Type, SanitizerSet SkippedChecks = SanitizerSet(), + llvm::Value *ArraySize = nullptr) { + if (!sanitizePerformTypeCheck()) + return; + EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(), + SkippedChecks, ArraySize); + } + + void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr, + QualType Type, CharUnits Alignment = CharUnits::Zero(), + SanitizerSet SkippedChecks = SanitizerSet(), + llvm::Value *ArraySize = nullptr) { + if (!sanitizePerformTypeCheck()) + return; + EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment, + SkippedChecks, ArraySize); + } + /// Emit a check that \p V is the address of storage of the /// appropriate size and alignment for an object of type \p Type /// (or if ArraySize is provided, for an array of that bound). @@ -3077,12 +3295,12 @@ public: llvm::Value *Index, QualType IndexType, QualType IndexedType, bool Accessed); - // Find a struct's flexible array member. It may be embedded inside multiple - // sub-structs, but must still be the last field. - const FieldDecl *FindFlexibleArrayMemberField(ASTContext &Ctx, - const RecordDecl *RD, - StringRef Name, - uint64_t &Offset); + // Find a struct's flexible array member and get its offset. It may be + // embedded inside multiple sub-structs, but must still be the last field. + const FieldDecl * + FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, + const FieldDecl *FAMDecl, + uint64_t &Offset); /// Find the FieldDecl specified in a FAM's "counted_by" attribute. Returns /// \p nullptr if either the attribute or the field doesn't exist. @@ -3160,17 +3378,17 @@ public: /// Address with original alloca instruction. Invalid if the variable was /// emitted as a global constant. - Address AllocaAddr; + RawAddress AllocaAddr; struct Invalid {}; AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()), - AllocaAddr(Address::invalid()) {} + AllocaAddr(RawAddress::invalid()) {} AutoVarEmission(const VarDecl &variable) : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr), IsEscapingByRef(false), IsConstantAggregate(false), - SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {} + SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {} bool wasEmittedAsGlobal() const { return !Addr.isValid(); } @@ -3193,7 +3411,7 @@ public: } /// Returns the address for the original alloca instruction. - Address getOriginalAllocatedAddress() const { return AllocaAddr; } + RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; } /// Returns the address of the object within this declaration. /// Note that this does not chase the forwarding pointer for @@ -3223,23 +3441,32 @@ public: llvm::GlobalValue::LinkageTypes Linkage); class ParamValue { - llvm::Value *Value; - llvm::Type *ElementType; - unsigned Alignment; - ParamValue(llvm::Value *V, llvm::Type *T, unsigned A) - : Value(V), ElementType(T), Alignment(A) {} + union { + Address Addr; + llvm::Value *Value; + }; + + bool IsIndirect; + + ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {} + ParamValue(Address A) : Addr(A), IsIndirect(true) {} + public: static ParamValue forDirect(llvm::Value *value) { - return ParamValue(value, nullptr, 0); + return ParamValue(value); } static ParamValue forIndirect(Address addr) { assert(!addr.getAlignment().isZero()); - return ParamValue(addr.getPointer(), addr.getElementType(), - addr.getAlignment().getQuantity()); + return ParamValue(addr); } - bool isIndirect() const { return Alignment != 0; } - llvm::Value *getAnyValue() const { return Value; } + bool isIndirect() const { return IsIndirect; } + llvm::Value *getAnyValue() const { + if (!isIndirect()) + return Value; + assert(!Addr.hasOffset() && "unexpected offset"); + return Addr.getBasePointer(); + } llvm::Value *getDirectValue() const { assert(!isIndirect()); @@ -3248,8 +3475,7 @@ public: Address getIndirectAddress() const { assert(isIndirect()); - return Address(Value, ElementType, CharUnits::fromQuantity(Alignment), - KnownNonNull); + return Addr; } }; @@ -3594,6 +3820,8 @@ public: void EmitOMPSimdDirective(const OMPSimdDirective &S); void EmitOMPTileDirective(const OMPTileDirective &S); void EmitOMPUnrollDirective(const OMPUnrollDirective &S); + void EmitOMPReverseDirective(const OMPReverseDirective &S); + void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S); void EmitOMPForDirective(const OMPForDirective &S); void EmitOMPForSimdDirective(const OMPForSimdDirective &S); void EmitOMPSectionsDirective(const OMPSectionsDirective &S); @@ -3808,6 +4036,8 @@ private: Expr *NextLB = nullptr; /// Update of UB after a whole chunk has been executed Expr *NextUB = nullptr; + /// Distinguish between the for distribute and sections + OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown; OMPLoopArguments() = default; OMPLoopArguments(Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk = nullptr, Expr *EUB = nullptr, @@ -3837,6 +4067,22 @@ private: void EmitSections(const OMPExecutableDirective &S); public: + //===--------------------------------------------------------------------===// + // OpenACC Emission + //===--------------------------------------------------------------------===// + void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) { + // TODO OpenACC: Implement this. It is currently implemented as a 'no-op', + // simply emitting its structured block, but in the future we will implement + // some sort of IR. + EmitStmt(S.getStructuredBlock()); + } + + void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) { + // TODO OpenACC: Implement this. It is currently implemented as a 'no-op', + // simply emitting its loop, but in the future we will implement + // some sort of IR. + EmitStmt(S.getLoop()); + } //===--------------------------------------------------------------------===// // LValue Expression Emission @@ -3986,6 +4232,11 @@ public: RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc); RValue EmitLoadOfGlobalRegLValue(LValue LV); + /// Like EmitLoadOfLValue but also handles complex and aggregate types. + RValue EmitLoadOfAnyValue(LValue V, + AggValueSlot Slot = AggValueSlot::ignored(), + SourceLocation Loc = {}); + /// EmitStoreThroughLValue - Store the specified rvalue into the specified /// lvalue, where both are guaranteed to the have the same type, and that type /// is 'Ty'. @@ -4023,8 +4274,8 @@ public: LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E, bool Accessed = false); LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E); - LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, - bool IsLowerBound = true); + LValue EmitArraySectionExpr(const ArraySectionExpr *E, + bool IsLowerBound = true); LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E); LValue EmitMemberExpr(const MemberExpr *E); LValue EmitObjCIsaExpr(const ObjCIsaExpr *E); @@ -4123,7 +4374,8 @@ public: RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, - SourceLocation Loc); + SourceLocation Loc, + bool IsVirtualFunctionPointerThunk = false); RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke = nullptr, @@ -4149,6 +4401,9 @@ public: llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name = ""); llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, + ArrayRef<Address> args, + const Twine &name = ""); + llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args, const Twine &name = ""); @@ -4174,6 +4429,45 @@ public: CXXDtorType Type, const CXXRecordDecl *RD); + bool isPointerKnownNonNull(const Expr *E); + + /// Create the discriminator from the storage address and the entity hash. + llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress, + llvm::Value *Discriminator); + CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema, + llvm::Value *StorageAddress, + GlobalDecl SchemaDecl, + QualType SchemaType); + + llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info, + llvm::Value *Pointer); + + llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info, + llvm::Value *Pointer); + + llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType, + const CGPointerAuthInfo &CurAuthInfo, + const CGPointerAuthInfo &NewAuthInfo, + bool IsKnownNonNull); + llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer, + const CGPointerAuthInfo &CurInfo, + const CGPointerAuthInfo &NewInfo); + + void EmitPointerAuthOperandBundle( + const CGPointerAuthInfo &Info, + SmallVectorImpl<llvm::OperandBundleDef> &Bundles); + + llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr, + QualType SourceType, QualType DestType); + Address authPointerToPointerCast(Address Ptr, QualType SourceType, + QualType DestType); + + Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy); + + llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) { + return getAsNaturalAddressOf(Addr, PointeeType).getBasePointer(); + } + // Return the copy constructor name with the prefix "__copy_constructor_" // removed. static std::string getNonTrivialCopyConstructorStr(QualType QT, @@ -4392,6 +4686,7 @@ public: llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E); llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E); @@ -4401,6 +4696,9 @@ public: llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue); + + void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst, + const CallExpr *E); void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID); @@ -4533,9 +4831,17 @@ public: /// aggregate type into a temporary LValue. LValue EmitAggExprToLValue(const Expr *E); - /// Build all the stores needed to initialize an aggregate at Dest with the - /// value Val. - void EmitAggregateStore(llvm::Value *Val, Address Dest, bool DestIsVolatile); + enum ExprValueKind { EVK_RValue, EVK_NonRValue }; + + /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into + /// destination address. + void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src, + ExprValueKind SrcKind); + + /// Create a store to \arg DstPtr from \arg Src, truncating the stored value + /// to at most \arg DstSize bytes. + void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize, + bool DstIsVolatile); /// EmitExtendGCLifetime - Given a pointer to an Objective-C object, /// make sure it survives garbage collection until this point. @@ -4581,7 +4887,7 @@ public: void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV, bool PerformInit); - llvm::Function *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor, + llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor, llvm::Constant *Addr); llvm::Function *createTLSAtExitStub(const VarDecl &VD, @@ -4745,6 +5051,11 @@ public: SourceLocation Loc, const Twine &Name = ""); + Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList, + llvm::Type *elementType, bool SignedIndices, + bool IsSubtraction, SourceLocation Loc, + CharUnits Align, const Twine &Name = ""); + /// Specifies which type of sanitizer check to apply when handling a /// particular builtin. enum BuiltinCheckKind { @@ -4807,6 +5118,10 @@ public: void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum); + void EmitNonNullArgCheck(Address Addr, QualType ArgType, + SourceLocation ArgLoc, AbstractCallee AC, + unsigned ParmNum); + /// EmitCallArg - Emit a single call argument. void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType); @@ -4835,6 +5150,29 @@ public: llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec, unsigned NumElementsDst, const llvm::Twine &Name = ""); + // Adds a convergence_ctrl token to |Input| and emits the required parent + // convergence instructions. + template <typename CallType> + CallType *addControlledConvergenceToken(CallType *Input) { + return cast<CallType>( + addConvergenceControlToken(Input, ConvergenceTokenStack.back())); + } + +private: + // Emits a convergence_loop instruction for the given |BB|, with |ParentToken| + // as it's parent convergence instr. + llvm::IntrinsicInst *emitConvergenceLoopToken(llvm::BasicBlock *BB, + llvm::Value *ParentToken); + // Adds a convergence_ctrl token with |ParentToken| as parent convergence + // instr to the call |Input|. + llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input, + llvm::Value *ParentToken); + // Find the convergence_entry instruction |F|, or emits ones if none exists. + // Returns the convergence instruction. + llvm::IntrinsicInst *getOrEmitConvergenceEntryToken(llvm::Function *F); + // Find the convergence_loop instruction for the loop defined by |LI|, or + // emits one if none exists. Returns the convergence instruction. + llvm::IntrinsicInst *getOrEmitConvergenceLoopToken(const LoopInfo *LI); private: llvm::MDNode *getRangeForLoadFromType(QualType Ty); @@ -5000,10 +5338,10 @@ private: llvm::Value *EmitAArch64CpuInit(); llvm::Value * FormAArch64ResolverCondition(const MultiVersionResolverOption &RO); + llvm::Value *EmitAArch64CpuSupports(const CallExpr *E); llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs); }; - inline DominatingLLVMValue::saved_type DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) { if (!needsSaving(value)) return saved_type(value, false); @@ -5015,7 +5353,7 @@ DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) { CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save"); CGF.Builder.CreateStore(value, alloca); - return saved_type(alloca.getPointer(), true); + return saved_type(alloca.emitRawPointer(CGF), true); } inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF, diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp index 1280bcd36de9..2a5d5f9083ae 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp @@ -30,6 +30,7 @@ #include "clang/AST/ASTContext.h" #include "clang/AST/ASTLambda.h" #include "clang/AST/CharUnits.h" +#include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclObjC.h" #include "clang/AST/DeclTemplate.h" @@ -52,6 +53,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/AttributeMask.h" #include "llvm/IR/CallingConv.h" @@ -69,8 +71,10 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/xxhash.h" +#include "llvm/TargetParser/RISCVISAInfo.h" #include "llvm/TargetParser/Triple.h" #include "llvm/TargetParser/X86TargetParser.h" +#include "llvm/Transforms/Utils/BuildLibCalls.h" #include <optional> using namespace clang; @@ -143,6 +147,10 @@ createTargetCodeGenInfo(CodeGenModule &CGM) { Kind = AArch64ABIKind::DarwinPCS; else if (Triple.isOSWindows()) return createWindowsAArch64TargetCodeGenInfo(CGM, AArch64ABIKind::Win64); + else if (Target.getABI() == "aapcs-soft") + Kind = AArch64ABIKind::AAPCSSoft; + else if (Target.getABI() == "pauthtest") + Kind = AArch64ABIKind::PAuthTest; return createAArch64TargetCodeGenInfo(CGM, Kind); } @@ -169,10 +177,7 @@ createTargetCodeGenInfo(CodeGenModule &CGM) { else if (ABIStr == "aapcs16") Kind = ARMABIKind::AAPCS16_VFP; else if (CodeGenOpts.FloatABI == "hard" || - (CodeGenOpts.FloatABI != "soft" && - (Triple.getEnvironment() == llvm::Triple::GNUEABIHF || - Triple.getEnvironment() == llvm::Triple::MuslEABIHF || - Triple.getEnvironment() == llvm::Triple::EABIHF))) + (CodeGenOpts.FloatABI != "soft" && Triple.isHardFloatABI())) Kind = ARMABIKind::AAPCS_VFP; return createARMTargetCodeGenInfo(CGM, Kind); @@ -335,10 +340,11 @@ CodeGenModule::CodeGenModule(ASTContext &C, : Context(C), LangOpts(C.getLangOpts()), FS(FS), HeaderSearchOpts(HSO), PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags), Target(C.getTargetInfo()), ABI(createCXXABI(*this)), - VMContext(M.getContext()), Types(*this), VTables(*this), + VMContext(M.getContext()), VTables(*this), SanitizerMD(new SanitizerMetadata(*this)) { // Initialize the type cache. + Types.reset(new CodeGenTypes(*this)); llvm::LLVMContext &LLVMContext = M.getContext(); VoidTy = llvm::Type::getVoidTy(LLVMContext); Int8Ty = llvm::Type::getInt8Ty(LLVMContext); @@ -362,7 +368,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth()); IntPtrTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getMaxPointerWidth()); - Int8PtrTy = llvm::PointerType::get(LLVMContext, 0); + Int8PtrTy = llvm::PointerType::get(LLVMContext, + C.getTargetAddressSpace(LangAS::Default)); const llvm::DataLayout &DL = M.getDataLayout(); AllocaInt8PtrTy = llvm::PointerType::get(LLVMContext, DL.getAllocaAddrSpace()); @@ -395,8 +402,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, // Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0. if (LangOpts.Sanitize.has(SanitizerKind::Thread) || (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0)) - TBAA.reset(new CodeGenTBAA(Context, TheModule, CodeGenOpts, getLangOpts(), - getCXXABI().getMangleContext())); + TBAA.reset(new CodeGenTBAA(Context, getTypes(), TheModule, CodeGenOpts, + getLangOpts())); // If debug info or coverage generation is enabled, create the CGDebugInfo // object. @@ -438,6 +445,11 @@ CodeGenModule::CodeGenModule(ASTContext &C, } ModuleNameHash = llvm::getUniqueInternalLinkagePostfix(Path); } + + // Record mregparm value now so it is visible through all of codegen. + if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86) + getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters", + CodeGenOpts.NumRegisterParameters); } CodeGenModule::~CodeGenModule() {} @@ -624,6 +636,26 @@ static bool checkAliasedGlobal( return true; } +// Emit a warning if toc-data attribute is requested for global variables that +// have aliases and remove the toc-data attribute. +static void checkAliasForTocData(llvm::GlobalVariable *GVar, + const CodeGenOptions &CodeGenOpts, + DiagnosticsEngine &Diags, + SourceLocation Location) { + if (GVar->hasAttribute("toc-data")) { + auto GVId = GVar->getName(); + // Is this a global variable specified by the user as local? + if ((llvm::binary_search(CodeGenOpts.TocDataVarsUserSpecified, GVId))) { + Diags.Report(Location, diag::warn_toc_unsupported_type) + << GVId << "the variable has an alias"; + } + llvm::AttributeSet CurrAttributes = GVar->getAttributes(); + llvm::AttributeSet NewAttributes = + CurrAttributes.removeAttribute(GVar->getContext(), "toc-data"); + GVar->setAttributes(NewAttributes); + } +} + void CodeGenModule::checkAliases() { // Check if the constructed aliases are well formed. It is really unfortunate // that we have to do this in CodeGen, but we only construct mangled names @@ -650,6 +682,12 @@ void CodeGenModule::checkAliases() { continue; } + if (getContext().getTargetInfo().getTriple().isOSAIX()) + if (const llvm::GlobalVariable *GVar = + dyn_cast<const llvm::GlobalVariable>(GV)) + checkAliasForTocData(const_cast<llvm::GlobalVariable *>(GVar), + getCodeGenOpts(), Diags, Location); + llvm::Constant *Aliasee = IsIFunc ? cast<llvm::GlobalIFunc>(Alias)->getResolver() : cast<llvm::GlobalAlias>(Alias)->getAliasee(); @@ -685,6 +723,11 @@ void CodeGenModule::checkAliases() { cast<llvm::GlobalAlias>(Alias)->setAliasee(Aliasee); } } + // ifunc resolvers are usually implemented to run before sanitizer + // initialization. Disable instrumentation to prevent the ordering issue. + if (IsIFunc) + cast<llvm::Function>(Aliasee)->addFnAttr( + llvm::Attribute::DisableSanitizerInstrumentation); } if (!Error) return; @@ -836,10 +879,6 @@ void CodeGenModule::Release() { AddGlobalCtor(CudaCtorFunction); } if (OpenMPRuntime) { - if (llvm::Function *OpenMPRequiresDirectiveRegFun = - OpenMPRuntime->emitRequiresDirectiveRegFun()) { - AddGlobalCtor(OpenMPRequiresDirectiveRegFun, 0); - } OpenMPRuntime->createOffloadEntriesAndInfoMetadata(); OpenMPRuntime->clear(); } @@ -860,6 +899,7 @@ void CodeGenModule::Release() { checkAliases(); EmitDeferredUnusedCoverageMappings(); CodeGenPGO(*this).setValueProfilingFlag(getModule()); + CodeGenPGO(*this).setProfileVersion(getModule()); if (CoverageMapping) CoverageMapping->emit(); if (CodeGenOpts.SanitizeCfiCrossDso) { @@ -872,13 +912,14 @@ void CodeGenModule::Release() { if (Context.getTargetInfo().getTriple().isWasm()) EmitMainVoidAlias(); - if (getTriple().isAMDGPU()) { - // Emit amdgpu_code_object_version module flag, which is code object version + if (getTriple().isAMDGPU() || + (getTriple().isSPIRV() && getTriple().getVendor() == llvm::Triple::AMD)) { + // Emit amdhsa_code_object_version module flag, which is code object version // times 100. if (getTarget().getTargetOpts().CodeObjectVersion != llvm::CodeObjectVersionKind::COV_None) { getModule().addModuleFlag(llvm::Module::Error, - "amdgpu_code_object_version", + "amdhsa_code_object_version", getTarget().getTargetOpts().CodeObjectVersion); } @@ -917,7 +958,15 @@ void CodeGenModule::Release() { llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external"); addCompilerUsedGlobal(GV); } - + if (LangOpts.HIP && !getLangOpts().OffloadingNewDriver) { + // Emit a unique ID so that host and device binaries from the same + // compilation unit can be associated. + auto *GV = new llvm::GlobalVariable( + getModule(), Int8Ty, false, llvm::GlobalValue::ExternalLinkage, + llvm::Constant::getNullValue(Int8Ty), + "__hip_cuid_" + getContext().getCUIDHash()); + addCompilerUsedGlobal(GV); + } emitLLVMUsed(); if (SanStats) SanStats->finish(); @@ -945,11 +994,6 @@ void CodeGenModule::Release() { NMD->addOperand(MD); } - // Record mregparm value now so it is visible through rest of codegen. - if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86) - getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters", - CodeGenOpts.NumRegisterParameters); - if (CodeGenOpts.DwarfVersion) { getModule().addModuleFlag(llvm::Module::Max, "Dwarf Version", CodeGenOpts.DwarfVersion); @@ -1044,21 +1088,31 @@ void CodeGenModule::Release() { llvm::MDString::get(VMContext, "ascii")); } - llvm::Triple::ArchType Arch = Context.getTargetInfo().getTriple().getArch(); - if ( Arch == llvm::Triple::arm - || Arch == llvm::Triple::armeb - || Arch == llvm::Triple::thumb - || Arch == llvm::Triple::thumbeb) { + llvm::Triple T = Context.getTargetInfo().getTriple(); + if (T.isARM() || T.isThumb()) { // The minimum width of an enum in bytes uint64_t EnumWidth = Context.getLangOpts().ShortEnums ? 1 : 4; getModule().addModuleFlag(llvm::Module::Error, "min_enum_size", EnumWidth); } - if (Arch == llvm::Triple::riscv32 || Arch == llvm::Triple::riscv64) { + if (T.isRISCV()) { StringRef ABIStr = Target.getABI(); llvm::LLVMContext &Ctx = TheModule.getContext(); getModule().addModuleFlag(llvm::Module::Error, "target-abi", llvm::MDString::get(Ctx, ABIStr)); + + // Add the canonical ISA string as metadata so the backend can set the ELF + // attributes correctly. We use AppendUnique so LTO will keep all of the + // unique ISA strings that were linked together. + const std::vector<std::string> &Features = + getTarget().getTargetOpts().Features; + auto ParseResult = + llvm::RISCVISAInfo::parseFeatures(T.isRISCV64() ? 64 : 32, Features); + if (!errorToBool(ParseResult.takeError())) + getModule().addModuleFlag( + llvm::Module::AppendUnique, "riscv-isa", + llvm::MDNode::get( + Ctx, llvm::MDString::get(Ctx, (*ParseResult)->toString()))); } if (CodeGenOpts.SanitizeCfiCrossDso) { @@ -1080,6 +1134,11 @@ void CodeGenModule::Release() { CodeGenOpts.SanitizeCfiCanonicalJumpTables); } + if (CodeGenOpts.SanitizeCfiICallNormalizeIntegers) { + getModule().addModuleFlag(llvm::Module::Override, "cfi-normalize-integers", + 1); + } + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) { getModule().addModuleFlag(llvm::Module::Override, "kcfi", 1); // KCFI assumes patchable-function-prefix is the same for all indirectly @@ -1127,10 +1186,7 @@ void CodeGenModule::Release() { getModule().addModuleFlag(llvm::Module::Override, "tag-stack-memory-buildattr", 1); - if (Arch == llvm::Triple::thumb || Arch == llvm::Triple::thumbeb || - Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb || - Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_32 || - Arch == llvm::Triple::aarch64_be) { + if (T.isARM() || T.isThumb() || T.isAArch64()) { if (LangOpts.BranchTargetEnforcement) getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement", 1); @@ -1147,6 +1203,37 @@ void CodeGenModule::Release() { if (!LangOpts.isSignReturnAddressWithAKey()) getModule().addModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 1); + + if (getTriple().isOSLinux()) { + assert(getTriple().isOSBinFormatELF()); + using namespace llvm::ELF; + uint64_t PAuthABIVersion = + (LangOpts.PointerAuthIntrinsics + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INTRINSICS) | + (LangOpts.PointerAuthCalls + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_CALLS) | + (LangOpts.PointerAuthReturns + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_RETURNS) | + (LangOpts.PointerAuthAuthTraps + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_AUTHTRAPS) | + (LangOpts.PointerAuthVTPtrAddressDiscrimination + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRADDRDISCR) | + (LangOpts.PointerAuthVTPtrTypeDiscrimination + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRTYPEDISCR) | + (LangOpts.PointerAuthInitFini + << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI); + static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI == + AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST, + "Update when new enum items are defined"); + if (PAuthABIVersion != 0) { + getModule().addModuleFlag(llvm::Module::Error, + "aarch64-elf-pauthabi-platform", + AARCH64_PAUTH_PLATFORM_LLVM_LINUX); + getModule().addModuleFlag(llvm::Module::Error, + "aarch64-elf-pauthabi-version", + PAuthABIVersion); + } + } } if (CodeGenOpts.StackClashProtector) @@ -1252,6 +1339,9 @@ void CodeGenModule::Release() { case CodeGenOptions::FramePointerKind::None: // 0 ("none") is the default. break; + case CodeGenOptions::FramePointerKind::Reserved: + getModule().setFramePointer(llvm::FramePointerKind::Reserved); + break; case CodeGenOptions::FramePointerKind::NonLeaf: getModule().setFramePointer(llvm::FramePointerKind::NonLeaf); break; @@ -1314,22 +1404,45 @@ void CodeGenModule::Release() { // that might affect the DLL storage class or the visibility, and // before anything that might act on these. setVisibilityFromDLLStorageClass(LangOpts, getModule()); + + // Check the tail call symbols are truly undefined. + if (getTriple().isPPC() && !MustTailCallUndefinedGlobals.empty()) { + for (auto &I : MustTailCallUndefinedGlobals) { + if (!I.first->isDefined()) + getDiags().Report(I.second, diag::err_ppc_impossible_musttail) << 2; + else { + StringRef MangledName = getMangledName(GlobalDecl(I.first)); + llvm::GlobalValue *Entry = GetGlobalValue(MangledName); + if (!Entry || Entry->isWeakForLinker() || + Entry->isDeclarationForLinker()) + getDiags().Report(I.second, diag::err_ppc_impossible_musttail) << 2; + } + } + } } void CodeGenModule::EmitOpenCLMetadata() { // SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the // opencl.ocl.version named metadata node. - // C++ for OpenCL has a distinct mapping for versions compatibile with OpenCL. - auto Version = LangOpts.getOpenCLCompatibleVersion(); - llvm::Metadata *OCLVerElts[] = { - llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( - Int32Ty, Version / 100)), - llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( - Int32Ty, (Version % 100) / 10))}; - llvm::NamedMDNode *OCLVerMD = - TheModule.getOrInsertNamedMetadata("opencl.ocl.version"); - llvm::LLVMContext &Ctx = TheModule.getContext(); - OCLVerMD->addOperand(llvm::MDNode::get(Ctx, OCLVerElts)); + // C++ for OpenCL has a distinct mapping for versions compatible with OpenCL. + auto CLVersion = LangOpts.getOpenCLCompatibleVersion(); + + auto EmitVersion = [this](StringRef MDName, int Version) { + llvm::Metadata *OCLVerElts[] = { + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int32Ty, Version / 100)), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int32Ty, (Version % 100) / 10))}; + llvm::NamedMDNode *OCLVerMD = TheModule.getOrInsertNamedMetadata(MDName); + llvm::LLVMContext &Ctx = TheModule.getContext(); + OCLVerMD->addOperand(llvm::MDNode::get(Ctx, OCLVerElts)); + }; + + EmitVersion("opencl.ocl.version", CLVersion); + if (LangOpts.OpenCLCPlusPlus) { + // In addition to the OpenCL compatible version, emit the C++ version. + EmitVersion("opencl.cxx.version", LangOpts.OpenCLCPlusPlusVersion); + } } void CodeGenModule::EmitBackendOptionsMetadata( @@ -1342,12 +1455,12 @@ void CodeGenModule::EmitBackendOptionsMetadata( void CodeGenModule::UpdateCompletedType(const TagDecl *TD) { // Make sure that this type is translated. - Types.UpdateCompletedType(TD); + getTypes().UpdateCompletedType(TD); } void CodeGenModule::RefreshTypeCacheForClass(const CXXRecordDecl *RD) { // Make sure that this type is translated. - Types.RefreshTypeCacheForClass(RD); + getTypes().RefreshTypeCacheForClass(RD); } llvm::MDNode *CodeGenModule::getTBAATypeInfo(QualType QTy) { @@ -1713,59 +1826,6 @@ static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM, Out << ".resolver"; } -static void AppendTargetVersionMangling(const CodeGenModule &CGM, - const TargetVersionAttr *Attr, - raw_ostream &Out) { - if (Attr->isDefaultVersion()) { - Out << ".default"; - return; - } - Out << "._"; - const TargetInfo &TI = CGM.getTarget(); - llvm::SmallVector<StringRef, 8> Feats; - Attr->getFeatures(Feats); - llvm::stable_sort(Feats, [&TI](const StringRef FeatL, const StringRef FeatR) { - return TI.multiVersionSortPriority(FeatL) < - TI.multiVersionSortPriority(FeatR); - }); - for (const auto &Feat : Feats) { - Out << 'M'; - Out << Feat; - } -} - -static void AppendTargetMangling(const CodeGenModule &CGM, - const TargetAttr *Attr, raw_ostream &Out) { - if (Attr->isDefaultVersion()) - return; - - Out << '.'; - const TargetInfo &Target = CGM.getTarget(); - ParsedTargetAttr Info = Target.parseTargetAttr(Attr->getFeaturesStr()); - llvm::sort(Info.Features, [&Target](StringRef LHS, StringRef RHS) { - // Multiversioning doesn't allow "no-${feature}", so we can - // only have "+" prefixes here. - assert(LHS.starts_with("+") && RHS.starts_with("+") && - "Features should always have a prefix."); - return Target.multiVersionSortPriority(LHS.substr(1)) > - Target.multiVersionSortPriority(RHS.substr(1)); - }); - - bool IsFirst = true; - - if (!Info.CPU.empty()) { - IsFirst = false; - Out << "arch_" << Info.CPU; - } - - for (StringRef Feat : Info.Features) { - if (!IsFirst) - Out << '_'; - IsFirst = false; - Out << Feat.substr(1); - } -} - // Returns true if GD is a function decl with internal linkage and // needs a unique suffix after the mangled name. static bool isUniqueInternalLinkageDecl(GlobalDecl GD, @@ -1775,41 +1835,6 @@ static bool isUniqueInternalLinkageDecl(GlobalDecl GD, (CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage); } -static void AppendTargetClonesMangling(const CodeGenModule &CGM, - const TargetClonesAttr *Attr, - unsigned VersionIndex, - raw_ostream &Out) { - const TargetInfo &TI = CGM.getTarget(); - if (TI.getTriple().isAArch64()) { - StringRef FeatureStr = Attr->getFeatureStr(VersionIndex); - if (FeatureStr == "default") { - Out << ".default"; - return; - } - Out << "._"; - SmallVector<StringRef, 8> Features; - FeatureStr.split(Features, "+"); - llvm::stable_sort(Features, - [&TI](const StringRef FeatL, const StringRef FeatR) { - return TI.multiVersionSortPriority(FeatL) < - TI.multiVersionSortPriority(FeatR); - }); - for (auto &Feat : Features) { - Out << 'M'; - Out << Feat; - } - } else { - Out << '.'; - StringRef FeatureStr = Attr->getFeatureStr(VersionIndex); - if (FeatureStr.starts_with("arch=")) - Out << "arch_" << FeatureStr.substr(sizeof("arch=") - 1); - else - Out << FeatureStr; - - Out << '.' << Attr->getMangledIndex(VersionIndex); - } -} - static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, const NamedDecl *ND, bool OmitMultiVersionMangling = false) { @@ -1863,16 +1888,31 @@ static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, FD->getAttr<CPUSpecificAttr>(), GD.getMultiVersionIndex(), Out); break; - case MultiVersionKind::Target: - AppendTargetMangling(CGM, FD->getAttr<TargetAttr>(), Out); + case MultiVersionKind::Target: { + auto *Attr = FD->getAttr<TargetAttr>(); + assert(Attr && "Expected TargetAttr to be present " + "for attribute mangling"); + const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo(); + Info.appendAttributeMangling(Attr, Out); break; - case MultiVersionKind::TargetVersion: - AppendTargetVersionMangling(CGM, FD->getAttr<TargetVersionAttr>(), Out); + } + case MultiVersionKind::TargetVersion: { + auto *Attr = FD->getAttr<TargetVersionAttr>(); + assert(Attr && "Expected TargetVersionAttr to be present " + "for attribute mangling"); + const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo(); + Info.appendAttributeMangling(Attr, Out); break; - case MultiVersionKind::TargetClones: - AppendTargetClonesMangling(CGM, FD->getAttr<TargetClonesAttr>(), - GD.getMultiVersionIndex(), Out); + } + case MultiVersionKind::TargetClones: { + auto *Attr = FD->getAttr<TargetClonesAttr>(); + assert(Attr && "Expected TargetClonesAttr to be present " + "for attribute mangling"); + unsigned Index = GD.getMultiVersionIndex(); + const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo(); + Info.appendAttributeMangling(Attr, Index, Out); break; + } case MultiVersionKind::None: llvm_unreachable("None multiversion type isn't valid here"); } @@ -2124,6 +2164,14 @@ void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD, llvm::AttributeList PAL; ConstructAttributeList(F->getName(), Info, GD, PAL, CallingConv, /*AttrOnCallSite=*/false, IsThunk); + if (CallingConv == llvm::CallingConv::X86_VectorCall && + getTarget().getTriple().isWindowsArm64EC()) { + SourceLocation Loc; + if (const Decl *D = GD.getDecl()) + Loc = D->getLocation(); + + Error(Loc, "__vectorcall calling convention is not currently supported"); + } F->setAttributes(PAL); F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv)); } @@ -2414,7 +2462,7 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (auto *Attr = D->getAttr<ArmNewAttr>()) { if (Attr->isNewZA()) - B.addAttribute("aarch64_pstate_za_new"); + B.addAttribute("aarch64_new_za"); if (Attr->isNewZT0()) B.addAttribute("aarch64_new_zt0"); } @@ -2664,7 +2712,7 @@ void CodeGenModule::setNonAliasAttributes(GlobalDecl GD, addUsedGlobal(F); if (auto *SA = D->getAttr<PragmaClangTextSectionAttr>()) if (!D->getAttr<SectionAttr>()) - F->addFnAttr("implicit-section-name", SA->getName()); + F->setSection(SA->getName()); llvm::AttrBuilder Attrs(F->getContext()); if (GetCPUAndFeaturesAttributes(GD, Attrs)) { @@ -3486,6 +3534,9 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { // Implicit template instantiations may change linkage if they are later // explicitly instantiated, so they should not be emitted eagerly. return false; + // Defer until all versions have been semantically checked. + if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion()) + return false; } if (const auto *VD = dyn_cast<VarDecl>(Global)) { if (Context.getInlineVariableDefinitionKind(VD) == @@ -3661,6 +3712,19 @@ template <typename AttrT> static bool hasImplicitAttr(const ValueDecl *D) { return D->isImplicit(); } +bool CodeGenModule::shouldEmitCUDAGlobalVar(const VarDecl *Global) const { + assert(LangOpts.CUDA && "Should not be called by non-CUDA languages"); + // We need to emit host-side 'shadows' for all global + // device-side variables because the CUDA runtime needs their + // size and host-side address in order to provide access to + // their device-side incarnations. + return !LangOpts.CUDAIsDevice || Global->hasAttr<CUDADeviceAttr>() || + Global->hasAttr<CUDAConstantAttr>() || + Global->hasAttr<CUDASharedAttr>() || + Global->getType()->isCUDADeviceBuiltinSurfaceType() || + Global->getType()->isCUDADeviceBuiltinTextureType(); +} + void CodeGenModule::EmitGlobal(GlobalDecl GD) { const auto *Global = cast<ValueDecl>(GD.getDecl()); @@ -3685,36 +3749,27 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Non-constexpr non-lambda implicit host device functions are not emitted // unless they are used on device side. if (LangOpts.CUDA) { - if (LangOpts.CUDAIsDevice) { + assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) && + "Expected Variable or Function"); + if (const auto *VD = dyn_cast<VarDecl>(Global)) { + if (!shouldEmitCUDAGlobalVar(VD)) + return; + } else if (LangOpts.CUDAIsDevice) { const auto *FD = dyn_cast<FunctionDecl>(Global); if ((!Global->hasAttr<CUDADeviceAttr>() || - (LangOpts.OffloadImplicitHostDeviceTemplates && FD && + (LangOpts.OffloadImplicitHostDeviceTemplates && hasImplicitAttr<CUDAHostAttr>(FD) && hasImplicitAttr<CUDADeviceAttr>(FD) && !FD->isConstexpr() && !isLambdaCallOperator(FD) && !getContext().CUDAImplicitHostDeviceFunUsedByDevice.count(FD))) && !Global->hasAttr<CUDAGlobalAttr>() && - !Global->hasAttr<CUDAConstantAttr>() && - !Global->hasAttr<CUDASharedAttr>() && - !Global->getType()->isCUDADeviceBuiltinSurfaceType() && - !Global->getType()->isCUDADeviceBuiltinTextureType() && !(LangOpts.HIPStdPar && isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>())) return; - } else { - // We need to emit host-side 'shadows' for all global - // device-side variables because the CUDA runtime needs their - // size and host-side address in order to provide access to - // their device-side incarnations. - - // So device-only functions are the only things we skip. - if (isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>() && - Global->hasAttr<CUDADeviceAttr>()) - return; - - assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) && - "Expected Variable or Function"); - } + // Device-only functions are the only things we skip. + } else if (!Global->hasAttr<CUDAHostAttr>() && + Global->hasAttr<CUDADeviceAttr>()) + return; } if (LangOpts.OpenMP) { @@ -3745,7 +3800,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { - if (!FD->doesDeclarationForceExternallyVisibleDefinition()) + if (!FD->doesDeclarationForceExternallyVisibleDefinition() && + (!FD->isMultiVersion() || !getTarget().getTriple().isAArch64())) return; StringRef MangledName = getMangledName(GD); @@ -3976,9 +4032,20 @@ bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) { // behavior may break ABI compatibility of the current unit. if (const Module *M = F->getOwningModule(); M && M->getTopLevelModule()->isNamedModule() && - getContext().getCurrentNamedModule() != M->getTopLevelModule() && - !F->hasAttr<AlwaysInlineAttr>()) - return false; + getContext().getCurrentNamedModule() != M->getTopLevelModule()) { + // There are practices to mark template member function as always-inline + // and mark the template as extern explicit instantiation but not give + // the definition for member function. So we have to emit the function + // from explicitly instantiation with always-inline. + // + // See https://github.com/llvm/llvm-project/issues/86893 for details. + // + // TODO: Maybe it is better to give it a warning if we call a non-inline + // function from other module units which is marked as always-inline. + if (!F->isTemplateInstantiation() || !F->hasAttr<AlwaysInlineAttr>()) { + return false; + } + } if (F->hasAttr<NoInlineAttr>()) return false; @@ -4028,17 +4095,21 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD, auto *Spec = FD->getAttr<CPUSpecificAttr>(); for (unsigned I = 0; I < Spec->cpus_size(); ++I) EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); - } else if (FD->isTargetClonesMultiVersion()) { - auto *Clone = FD->getAttr<TargetClonesAttr>(); - for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I) - if (Clone->isFirstOfVersion(I)) + } else if (auto *TC = FD->getAttr<TargetClonesAttr>()) { + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) + // AArch64 favors the default target version over the clone if any. + if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) && + TC->isFirstOfVersion(I)) EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); // Ensure that the resolver function is also emitted. GetOrCreateMultiVersionResolver(GD); - } else if (FD->hasAttr<TargetVersionAttr>()) { - GetOrCreateMultiVersionResolver(GD); } else EmitGlobalFunctionDefinition(GD, GV); + + // Defer the resolver emission until we can reason whether the TU + // contains a default target version implementation. + if (FD->isTargetVersionMultiVersion()) + AddDeferredMultiVersionResolverToEmit(GD); } void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) { @@ -4131,105 +4202,92 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); - SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options; - if (FD->isTargetMultiVersion()) { - getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } - if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { - const auto *TA = CurFD->getAttr<TargetAttr>(); - llvm::SmallVector<StringRef, 8> Feats; - TA->getAddedFeatures(Feats); - Options.emplace_back(cast<llvm::Function>(Func), - TA->getArchitecture(), Feats); - } else { - const auto *TVA = CurFD->getAttr<TargetVersionAttr>(); - llvm::SmallVector<StringRef, 8> Feats; - TVA->getFeatures(Feats); - Options.emplace_back(cast<llvm::Function>(Func), - /*Architecture*/ "", Feats); - } - }); - } else if (FD->isTargetClonesMultiVersion()) { - const auto *TC = FD->getAttr<TargetClonesAttr>(); - for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); - ++VersionIndex) { - if (!TC->isFirstOfVersion(VersionIndex)) - continue; - GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), - VersionIndex}; - StringRef Version = TC->getFeatureStr(VersionIndex); - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (FD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(CurGD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); + auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) { + GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (Decl->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); } + assert(Func && "This should have just been created"); + } + return cast<llvm::Function>(Func); + }; - StringRef Architecture; - llvm::SmallVector<StringRef, 1> Feature; + // For AArch64, a resolver is only emitted if a function marked with + // target_version("default")) or target_clones() is present and defined + // in this TU. For other architectures it is always emitted. + bool ShouldEmitResolver = !getTarget().getTriple().isAArch64(); + SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options; - if (getTarget().getTriple().isAArch64()) { - if (Version != "default") { - llvm::SmallVector<StringRef, 8> VerFeats; - Version.split(VerFeats, "+"); - for (auto &CurFeat : VerFeats) - Feature.push_back(CurFeat.trim()); - } - } else { - if (Version.starts_with("arch=")) - Architecture = Version.drop_front(sizeof("arch=") - 1); - else if (Version != "default") - Feature.push_back(Version); - } + getContext().forEachMultiversionedFunctionVersion( + FD, [&](const FunctionDecl *CurFD) { + llvm::SmallVector<StringRef, 8> Feats; + bool IsDefined = CurFD->doesThisDeclarationHaveABody(); + + if (const auto *TA = CurFD->getAttr<TargetAttr>()) { + TA->getAddedFeatures(Feats); + llvm::Function *Func = createFunction(CurFD); + Options.emplace_back(Func, TA->getArchitecture(), Feats); + } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) { + if (TVA->isDefaultVersion() && IsDefined) + ShouldEmitResolver = true; + TVA->getFeatures(Feats); + llvm::Function *Func = createFunction(CurFD); + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) { + if (IsDefined) + ShouldEmitResolver = true; + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) { + if (!TC->isFirstOfVersion(I)) + continue; + + llvm::Function *Func = createFunction(CurFD, I); + StringRef Architecture; + Feats.clear(); + if (getTarget().getTriple().isAArch64()) + TC->getFeatures(Feats, I); + else { + StringRef Version = TC->getFeatureStr(I); + if (Version.starts_with("arch=")) + Architecture = Version.drop_front(sizeof("arch=") - 1); + else if (Version != "default") + Feats.push_back(Version); + } + Options.emplace_back(Func, Architecture, Feats); + } + } else + llvm_unreachable("unexpected MultiVersionKind"); + }); - Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature); - } - } else { - assert(0 && "Expected a target or target_clones multiversion function"); + if (!ShouldEmitResolver) continue; - } llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); - if (FD->isTargetClonesMultiVersion()) { - const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + if (FD->isTargetClonesMultiVersion() && + !getTarget().getTriple().isAArch64()) { std::string MangledName = getMangledNameImpl( *this, GD, FD, /*OmitMultiVersionMangling=*/true); - // In prior versions of Clang, the mangling for ifuncs incorrectly - // included an .ifunc suffix. This alias is generated for backward - // compatibility. It is deprecated, and may be removed in the future. - auto *Alias = llvm::GlobalAlias::create( - DeclTy, 0, getMultiversionLinkage(*this, GD), - MangledName + ".ifunc", IFunc, &getModule()); - SetCommonAttributes(FD, Alias); + if (!GetGlobalValue(MangledName + ".ifunc")) { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + // In prior versions of Clang, the mangling for ifuncs incorrectly + // included an .ifunc suffix. This alias is generated for backward + // compatibility. It is deprecated, and may be removed in the future. + auto *Alias = llvm::GlobalAlias::create( + DeclTy, 0, getMultiversionLinkage(*this, GD), + MangledName + ".ifunc", IFunc, &getModule()); + SetCommonAttributes(FD, Alias); + } } } llvm::Function *ResolverFunc = cast<llvm::Function>(ResolverConstant); @@ -4262,6 +4320,14 @@ void CodeGenModule::emitMultiVersionFunctions() { emitMultiVersionFunctions(); } +static void replaceDeclarationWith(llvm::GlobalValue *Old, + llvm::Constant *New) { + assert(cast<llvm::Function>(Old)->isDeclaration() && "Not a declaration"); + New->takeName(Old); + Old->replaceAllUsesWith(New); + Old->eraseFromParent(); +} + void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Not a FunctionDecl?"); @@ -4366,12 +4432,9 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { // Fix up function declarations that were created for cpu_specific before // cpu_dispatch was known if (!isa<llvm::GlobalIFunc>(IFunc)) { - assert(cast<llvm::Function>(IFunc)->isDeclaration()); auto *GI = llvm::GlobalIFunc::create(DeclTy, 0, Linkage, "", ResolverFunc, &getModule()); - GI->takeName(IFunc); - IFunc->replaceAllUsesWith(GI); - IFunc->eraseFromParent(); + replaceDeclarationWith(IFunc, GI); IFunc = GI; } @@ -4386,8 +4449,23 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { } } +/// Adds a declaration to the list of multi version functions if not present. +void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + assert(FD && "Not a FunctionDecl?"); + + if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) { + std::string MangledName = + getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true); + if (!DeferredResolversToEmit.insert(MangledName).second) + return; + } + MultiVersionFuncs.push_back(GD); +} + /// If a dispatcher for the specified mangled name is not in the module, create -/// and return an llvm Function with the specified type. +/// and return it. The dispatcher is either an llvm Function with the specified +/// type, or a global ifunc. llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Not a FunctionDecl?"); @@ -4399,14 +4477,31 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { // a separate resolver). std::string ResolverName = MangledName; if (getTarget().supportsIFunc()) { - if (!FD->isTargetClonesMultiVersion()) + switch (FD->getMultiVersionKind()) { + case MultiVersionKind::None: + llvm_unreachable("unexpected MultiVersionKind::None for resolver"); + case MultiVersionKind::Target: + case MultiVersionKind::CPUSpecific: + case MultiVersionKind::CPUDispatch: ResolverName += ".ifunc"; + break; + case MultiVersionKind::TargetClones: + case MultiVersionKind::TargetVersion: + break; + } } else if (FD->isTargetMultiVersion()) { ResolverName += ".resolver"; } - // If the resolver has already been created, just return it. - if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName)) + // If the resolver has already been created, just return it. This lookup may + // yield a function declaration instead of a resolver on AArch64. That is + // because we didn't know whether a resolver will be generated when we first + // encountered a use of the symbol named after this resolver. Therefore, + // targets which support ifuncs should not return here unless we actually + // found an ifunc. + llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName); + if (ResolverGV && + (isa<llvm::GlobalIFunc>(ResolverGV) || !getTarget().supportsIFunc())) return ResolverGV; const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); @@ -4415,7 +4510,7 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { // The resolver needs to be created. For target and target_clones, defer // creation until the end of the TU. if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion()) - MultiVersionFuncs.push_back(GD); + AddDeferredMultiVersionResolverToEmit(GD); // For cpu_specific, don't create an ifunc yet because we don't know if the // cpu_dispatch will be emitted in this translation unit. @@ -4432,7 +4527,8 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { "", Resolver, &getModule()); GIF->setName(ResolverName); SetCommonAttributes(FD, GIF); - + if (ResolverGV) + replaceDeclarationWith(ResolverGV, GIF); return GIF; } @@ -4441,9 +4537,24 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { assert(isa<llvm::GlobalValue>(Resolver) && "Resolver should be created for the first time"); SetCommonAttributes(FD, cast<llvm::GlobalValue>(Resolver)); + if (ResolverGV) + replaceDeclarationWith(ResolverGV, Resolver); return Resolver; } +bool CodeGenModule::shouldDropDLLAttribute(const Decl *D, + const llvm::GlobalValue *GV) const { + auto SC = GV->getDLLStorageClass(); + if (SC == llvm::GlobalValue::DefaultStorageClass) + return false; + const Decl *MRD = D->getMostRecentDecl(); + return (((SC == llvm::GlobalValue::DLLImportStorageClass && + !MRD->hasAttr<DLLImportAttr>()) || + (SC == llvm::GlobalValue::DLLExportStorageClass && + !MRD->hasAttr<DLLExportAttr>())) && + !shouldMapVisibilityToDLLExport(cast<NamedDecl>(MRD))); +} + /// GetOrCreateLLVMFunction - If the specified mangled name is not in the /// module, create and return an llvm Function with the specified type. If there /// is something in the module with the specified name, return it potentially @@ -4457,6 +4568,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( ForDefinition_t IsForDefinition) { const Decl *D = GD.getDecl(); + std::string NameWithoutMultiVersionMangling; // Any attempts to use a MultiVersion function should result in retrieving // the iFunc instead. Name Mangling will handle the rest of the changes. if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D)) { @@ -4478,11 +4590,24 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( if (FD->isMultiVersion()) { UpdateMultiVersionNames(GD, FD, MangledName); - if (!IsForDefinition) - return GetOrCreateMultiVersionResolver(GD); + if (!IsForDefinition) { + // On AArch64 we do not immediatelly emit an ifunc resolver when a + // function is used. Instead we defer the emission until we see a + // default definition. In the meantime we just reference the symbol + // without FMV mangling (it may or may not be replaced later). + if (getTarget().getTriple().isAArch64()) { + AddDeferredMultiVersionResolverToEmit(GD); + NameWithoutMultiVersionMangling = getMangledNameImpl( + *this, GD, FD, /*OmitMultiVersionMangling=*/true); + } else + return GetOrCreateMultiVersionResolver(GD); + } } } + if (!NameWithoutMultiVersionMangling.empty()) + MangledName = NameWithoutMultiVersionMangling; + // Lookup the entry, lazily creating it if necessary. llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (Entry) { @@ -4493,8 +4618,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( } // Handle dropped DLL attributes. - if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() && - !shouldMapVisibilityToDLLExport(cast_or_null<NamedDecl>(D))) { + if (D && shouldDropDLLAttribute(D, Entry)) { Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); setDSOLocal(Entry); } @@ -4753,6 +4877,10 @@ CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name, } } setDSOLocal(F); + // FIXME: We should use CodeGenModule::SetLLVMFunctionAttributes() instead + // of trying to approximate the attributes using the LLVM function + // signature. This requires revising the API of CreateRuntimeFunction(). + markRegisterParameterAttributes(F); } } @@ -4784,8 +4912,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, } // Handle dropped DLL attributes. - if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() && - !shouldMapVisibilityToDLLExport(D)) + if (D && shouldDropDLLAttribute(D, Entry)) Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D) @@ -5090,8 +5217,11 @@ void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) { EmitGlobalVarDefinition(D); } -void CodeGenModule::EmitExternalDeclaration(const VarDecl *D) { - EmitExternalVarDeclaration(D); +void CodeGenModule::EmitExternalDeclaration(const DeclaratorDecl *D) { + if (auto const *V = dyn_cast<const VarDecl>(D)) + EmitExternalVarDeclaration(V); + if (auto const *FD = dyn_cast<const FunctionDecl>(D)) + EmitExternalFunctionDeclaration(FD); } CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { @@ -5249,6 +5379,10 @@ void CodeGenModule::maybeSetTrivialComdat(const Decl &D, GO.setComdat(TheModule.getOrInsertComdat(GO.getName())); } +const ABIInfo &CodeGenModule::getABIInfo() { + return getTargetCodeGenInfo().getABIInfo(); +} + /// Pass IsTentative as true if you want to create a tentative definition. void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, bool IsTentative) { @@ -5275,6 +5409,18 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, !IsDefinitionAvailableExternally && D->needsDestruction(getContext()) == QualType::DK_cxx_destructor; + // It is helpless to emit the definition for an available_externally variable + // which can't be marked as const. + // We don't need to check if it needs global ctor or dtor. See the above + // comment for ideas. + if (IsDefinitionAvailableExternally && + (!D->hasConstantInitialization() || + // TODO: Update this when we have interface to check constexpr + // destructor. + D->needsDestruction(getContext()) || + !D->getType().isConstantStorage(getContext(), true, true))) + return; + const VarDecl *InitDecl; const Expr *InitExpr = D->getAnyInitializer(InitDecl); @@ -5515,6 +5661,18 @@ void CodeGenModule::EmitExternalVarDeclaration(const VarDecl *D) { } } +void CodeGenModule::EmitExternalFunctionDeclaration(const FunctionDecl *FD) { + if (CGDebugInfo *DI = getModuleDebugInfo()) + if (getCodeGenOpts().hasReducedDebugInfo()) { + auto *Ty = getTypes().ConvertType(FD->getType()); + StringRef MangledName = getMangledName(FD); + auto *Fn = dyn_cast<llvm::Function>( + GetOrCreateLLVMFunction(MangledName, Ty, FD, /* ForVTable */ false)); + if (!Fn->getSubprogram()) + DI->EmitFunctionDecl(FD, FD->getLocation(), FD->getType(), Fn); + } +} + static bool isVarDeclStrongDefinition(const ASTContext &Context, CodeGenModule &CGM, const VarDecl *D, bool NoCommon) { @@ -5674,15 +5832,17 @@ CodeGenModule::getLLVMLinkageVarDefinition(const VarDecl *VD) { static void replaceUsesOfNonProtoConstant(llvm::Constant *old, llvm::Function *newFn) { // Fast path. - if (old->use_empty()) return; + if (old->use_empty()) + return; llvm::Type *newRetTy = newFn->getReturnType(); - SmallVector<llvm::Value*, 4> newArgs; + SmallVector<llvm::Value *, 4> newArgs; + + SmallVector<llvm::CallBase *> callSitesToBeRemovedFromParent; for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end(); - ui != ue; ) { - llvm::Value::use_iterator use = ui++; // Increment before the use is erased. - llvm::User *user = use->getUser(); + ui != ue; ui++) { + llvm::User *user = ui->getUser(); // Recognize and replace uses of bitcasts. Most calls to // unprototyped functions will use bitcasts. @@ -5694,8 +5854,9 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, // Recognize calls to the function. llvm::CallBase *callSite = dyn_cast<llvm::CallBase>(user); - if (!callSite) continue; - if (!callSite->isCallee(&*use)) + if (!callSite) + continue; + if (!callSite->isCallee(&*ui)) continue; // If the return types don't match exactly, then we can't @@ -5764,6 +5925,10 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, if (callSite->getDebugLoc()) newCall->setDebugLoc(callSite->getDebugLoc()); + callSitesToBeRemovedFromParent.push_back(callSite); + } + + for (auto *callSite : callSitesToBeRemovedFromParent) { callSite->eraseFromParent(); } } @@ -5787,7 +5952,8 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old, void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) { auto DK = VD->isThisDeclarationADefinition(); - if (DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>()) + if ((DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>()) || + (LangOpts.CUDA && !shouldEmitCUDAGlobalVar(VD))) return; TemplateSpecializationKind TSK = VD->getTemplateSpecializationKind(); @@ -5959,11 +6125,14 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { Aliases.push_back(GD); - llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType()); - llvm::Type *ResolverTy = llvm::GlobalIFunc::getResolverFunctionType(DeclTy); + // The resolver might not be visited yet. Specify a dummy non-function type to + // indicate IsIncompleteFunction. Either the type is ignored (if the resolver + // was emitted) or the whole function will be replaced (if the resolver has + // not been emitted). llvm::Constant *Resolver = - GetOrCreateLLVMFunction(IFA->getResolver(), ResolverTy, {}, + GetOrCreateLLVMFunction(IFA->getResolver(), VoidTy, {}, /*ForVTable=*/false); + llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType()); llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create(DeclTy, 0, llvm::Function::ExternalLinkage, "", Resolver, &getModule()); @@ -5987,9 +6156,6 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { Entry->eraseFromParent(); } else GIF->setName(MangledName); - if (auto *F = dyn_cast<llvm::Function>(Resolver)) { - F->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation); - } SetCommonAttributes(GD, GIF); } @@ -6046,9 +6212,6 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { return ConstantAddress( C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment())); - llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); - llvm::Constant *Zeros[] = { Zero, Zero }; - const ASTContext &Context = getContext(); const llvm::Triple &Triple = getTriple(); @@ -6119,8 +6282,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { // Decay array -> ptr CFConstantStringClassRef = - IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty) - : llvm::ConstantExpr::getGetElementPtr(Ty, C, Zeros); + IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty) : C; } QualType CFTy = Context.getCFConstantStringType(); @@ -6176,10 +6338,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { GV->setSection(".rodata"); // String. - llvm::Constant *Str = - llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); - - Fields.add(Str); + Fields.add(GV); // String length. llvm::IntegerType *LengthTy = @@ -6273,7 +6432,7 @@ CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) { // Resize the string to the right size, which is indicated by its type. const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType()); assert(CAT && "String literal not of constant array type!"); - Str.resize(CAT->getSize().getZExtValue()); + Str.resize(CAT->getZExtSize()); return llvm::ConstantDataArray::getString(VMContext, Str, false); } @@ -6330,7 +6489,8 @@ GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, ConstantAddress CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, StringRef Name) { - CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(S->getType()); + CharUnits Alignment = + getContext().getAlignOfGlobalVarInChars(S->getType(), /*VD=*/nullptr); llvm::Constant *C = GetConstantArrayFromStringLiteral(S); llvm::GlobalVariable **Entry = nullptr; @@ -6393,8 +6553,8 @@ CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) { ConstantAddress CodeGenModule::GetAddrOfConstantCString( const std::string &Str, const char *GlobalName) { StringRef StrWithNull(Str.c_str(), Str.size() + 1); - CharUnits Alignment = - getContext().getAlignOfGlobalVarInChars(getContext().CharTy); + CharUnits Alignment = getContext().getAlignOfGlobalVarInChars( + getContext().CharTy, /*VD=*/nullptr); llvm::Constant *C = llvm::ConstantDataArray::getString(getLLVMContext(), StrWithNull, false); @@ -6609,7 +6769,7 @@ static bool AllTrivialInitializers(CodeGenModule &CGM, void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { // We might need a .cxx_destruct even if we don't have any ivar initializers. if (needsDestructMethod(D)) { - IdentifierInfo *II = &getContext().Idents.get(".cxx_destruct"); + const IdentifierInfo *II = &getContext().Idents.get(".cxx_destruct"); Selector cxxSelector = getContext().Selectors.getSelector(0, &II); ObjCMethodDecl *DTORMethod = ObjCMethodDecl::Create( getContext(), D->getLocation(), D->getLocation(), cxxSelector, @@ -6629,7 +6789,7 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { AllTrivialInitializers(*this, D)) return; - IdentifierInfo *II = &getContext().Idents.get(".cxx_construct"); + const IdentifierInfo *II = &getContext().Idents.get(".cxx_construct"); Selector cxxSelector = getContext().Selectors.getSelector(0, &II); // The constructor returns 'self'. ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create( @@ -6920,8 +7080,8 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { // For C++ standard modules we are done - we will call the module // initializer for imported modules, and that will likewise call those for // any imports it has. - if (CXX20ModuleInits && Import->getImportedOwningModule() && - !Import->getImportedOwningModule()->isModuleMapModule()) + if (CXX20ModuleInits && Import->getImportedModule() && + Import->getImportedModule()->isNamedModule()) break; // For clang C++ module map modules the initializers for sub-modules are @@ -7027,6 +7187,9 @@ void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) { SourceManager &SM = getContext().getSourceManager(); if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(D->getBeginLoc())) break; + if (!llvm::coverage::SystemHeadersCoverage && + SM.isInSystemHeader(D->getBeginLoc())) + break; DeferredEmptyCoverageMappingDecls.try_emplace(D, true); break; } @@ -7197,7 +7360,7 @@ void CodeGenModule::EmitStaticExternCAliases() { if (!getTargetCodeGenInfo().shouldEmitStaticExternCAliases()) return; for (auto &I : StaticExternCValues) { - IdentifierInfo *Name = I.first; + const IdentifierInfo *Name = I.first; llvm::GlobalValue *Val = I.second; // If Val is null, that implies there were multiple declarations that each @@ -7258,7 +7421,7 @@ void CodeGenFunction::EmitDeclMetadata() { for (auto &I : LocalDeclMap) { const Decl *D = I.first; - llvm::Value *Addr = I.second.getPointer(); + llvm::Value *Addr = I.second.emitRawPointer(*this); if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) { llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D); Alloca->setMetadata( @@ -7628,7 +7791,5 @@ void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { NewBuilder->WeakRefReferences = std::move(WeakRefReferences); - NewBuilder->TBAA = std::move(TBAA); - NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h index ec34680fd3f7..c58bb88035ca 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h @@ -24,7 +24,6 @@ #include "clang/AST/Mangle.h" #include "clang/Basic/ABI.h" #include "clang/Basic/LangOptions.h" -#include "clang/Basic/Module.h" #include "clang/Basic/NoSanitizeList.h" #include "clang/Basic/ProfileList.h" #include "clang/Basic/TargetInfo.h" @@ -70,6 +69,7 @@ class Expr; class Stmt; class StringLiteral; class NamedDecl; +class PointerAuthSchema; class ValueDecl; class VarDecl; class LangOptions; @@ -320,7 +320,7 @@ private: // This should not be moved earlier, since its initialization depends on some // of the previous reference members being already initialized and also checks // if TheTargetCodeGenInfo is NULL - CodeGenTypes Types; + std::unique_ptr<CodeGenTypes> Types; /// Holds information about C++ vtables. CodeGenVTables VTables; @@ -348,6 +348,8 @@ private: /// yet. llvm::DenseMap<StringRef, GlobalDecl> DeferredDecls; + llvm::StringSet<llvm::BumpPtrAllocator> DeferredResolversToEmit; + /// This is a list of deferred decls which we have seen that *are* actually /// referenced. These get code generated when the module is done. std::vector<GlobalDecl> DeferredDeclsToEmit; @@ -433,7 +435,7 @@ private: // Store deferred function annotations so they can be emitted at the end with // most up to date ValueDecl that will have all the inherited annotations. - llvm::DenseMap<StringRef, const ValueDecl *> DeferredAnnotations; + llvm::MapVector<StringRef, const ValueDecl *> DeferredAnnotations; /// Map used to get unique annotation strings. llvm::StringMap<llvm::Constant*> AnnotationStrings; @@ -483,6 +485,14 @@ private: typedef std::pair<OrderGlobalInitsOrStermFinalizers, llvm::Function *> GlobalInitData; + // When a tail call is performed on an "undefined" symbol, on PPC without pc + // relative feature, the tail call is not allowed. In "EmitCall" for such + // tail calls, the "undefined" symbols may be forward declarations, their + // definitions are provided in the module after the callsites. For such tail + // calls, diagnose message should not be emitted. + llvm::SmallSetVector<std::pair<const FunctionDecl *, SourceLocation>, 4> + MustTailCallUndefinedGlobals; + struct GlobalInitPriorityCmp { bool operator()(const GlobalInitData &LHS, const GlobalInitData &RHS) const { @@ -553,6 +563,9 @@ private: bool isTriviallyRecursive(const FunctionDecl *F); bool shouldEmitFunction(GlobalDecl GD); + // Whether a global variable should be emitted by CUDA/HIP host/device + // related attributes. + bool shouldEmitCUDAGlobalVar(const VarDecl *VD) const; bool shouldOpportunisticallyEmitVTables(); /// Map used to be sure we don't emit the same CompoundLiteral twice. llvm::DenseMap<const CompoundLiteralExpr *, llvm::GlobalVariable *> @@ -607,6 +620,13 @@ private: std::pair<std::unique_ptr<CodeGenFunction>, const TopLevelStmtDecl *> GlobalTopLevelStmtBlockInFlight; + llvm::DenseMap<GlobalDecl, uint16_t> PtrAuthDiscriminatorHashes; + + llvm::DenseMap<const CXXRecordDecl *, std::optional<PointerAuthQualifier>> + VTablePtrAuthInfos; + std::optional<PointerAuthQualifier> + computeVTPointerAuthentication(const CXXRecordDecl *ThisClass); + public: CodeGenModule(ASTContext &C, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, const HeaderSearchOptions &headersearchopts, @@ -756,6 +776,7 @@ public: bool supportsCOMDAT() const; void maybeSetTrivialComdat(const Decl &D, llvm::GlobalObject &GO); + const ABIInfo &getABIInfo(); CGCXXABI &getCXXABI() const { return *ABI; } llvm::LLVMContext &getLLVMContext() { return VMContext; } @@ -763,7 +784,7 @@ public: const TargetCodeGenInfo &getTargetCodeGenInfo(); - CodeGenTypes &getTypes() { return Types; } + CodeGenTypes &getTypes() { return *Types; } CodeGenVTables &getVTables() { return VTables; } @@ -936,11 +957,69 @@ public: // Return the function body address of the given function. llvm::Constant *GetFunctionStart(const ValueDecl *Decl); + /// Return a function pointer for a reference to the given function. + /// This correctly handles weak references, but does not apply a + /// pointer signature. + llvm::Constant *getRawFunctionPointer(GlobalDecl GD, + llvm::Type *Ty = nullptr); + + /// Return the ABI-correct function pointer value for a reference + /// to the given function. This will apply a pointer signature if + /// necessary, caching the result for the given function. + llvm::Constant *getFunctionPointer(GlobalDecl GD, llvm::Type *Ty = nullptr); + + /// Return the ABI-correct function pointer value for a reference + /// to the given function. This will apply a pointer signature if + /// necessary. + llvm::Constant *getFunctionPointer(llvm::Constant *Pointer, + QualType FunctionType); + + llvm::Constant *getMemberFunctionPointer(const FunctionDecl *FD, + llvm::Type *Ty = nullptr); + + llvm::Constant *getMemberFunctionPointer(llvm::Constant *Pointer, + QualType FT); + + CGPointerAuthInfo getFunctionPointerAuthInfo(QualType T); + + CGPointerAuthInfo getMemberFunctionPointerAuthInfo(QualType FT); + + CGPointerAuthInfo getPointerAuthInfoForPointeeType(QualType type); + + CGPointerAuthInfo getPointerAuthInfoForType(QualType type); + + bool shouldSignPointer(const PointerAuthSchema &Schema); + llvm::Constant *getConstantSignedPointer(llvm::Constant *Pointer, + const PointerAuthSchema &Schema, + llvm::Constant *StorageAddress, + GlobalDecl SchemaDecl, + QualType SchemaType); + + llvm::Constant * + getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key, + llvm::Constant *StorageAddress, + llvm::ConstantInt *OtherDiscriminator); + + llvm::ConstantInt * + getPointerAuthOtherDiscriminator(const PointerAuthSchema &Schema, + GlobalDecl SchemaDecl, QualType SchemaType); + + uint16_t getPointerAuthDeclDiscriminator(GlobalDecl GD); + std::optional<CGPointerAuthInfo> + getVTablePointerAuthInfo(CodeGenFunction *Context, + const CXXRecordDecl *Record, + llvm::Value *StorageAddress); + + std::optional<PointerAuthQualifier> + getVTablePointerAuthentication(const CXXRecordDecl *thisClass); + + CGPointerAuthInfo EmitPointerAuthInfo(const RecordDecl *RD); + // Return whether RTTI information should be emitted for this target. bool shouldEmitRTTI(bool ForEH = false) { return (ForEH || getLangOpts().RTTI) && !getLangOpts().CUDAIsDevice && !(getLangOpts().OpenMP && getLangOpts().OpenMPIsTargetDevice && - getTriple().isNVPTX()); + (getTriple().isNVPTX() || getTriple().isAMDGPU())); } /// Get the address of the RTTI descriptor for the given type. @@ -1239,6 +1318,9 @@ public: /// Return true iff the given type uses 'sret' when used as a return type. bool ReturnTypeUsesSRet(const CGFunctionInfo &FI); + /// Return true iff the given type has `inreg` set. + bool ReturnTypeHasInReg(const CGFunctionInfo &FI); + /// Return true iff the given type uses an argument slot when 'sret' is used /// as a return type. bool ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI); @@ -1280,7 +1362,7 @@ public: void EmitTentativeDefinition(const VarDecl *D); - void EmitExternalDeclaration(const VarDecl *D); + void EmitExternalDeclaration(const DeclaratorDecl *D); void EmitVTable(CXXRecordDecl *Class); @@ -1581,13 +1663,31 @@ public: void AddGlobalDtor(llvm::Function *Dtor, int Priority = 65535, bool IsDtorAttrFunc = false); + // Return whether structured convergence intrinsics should be generated for + // this target. + bool shouldEmitConvergenceTokens() const { + // TODO: this should probably become unconditional once the controlled + // convergence becomes the norm. + return getTriple().isSPIRVLogical(); + } + + void addUndefinedGlobalForTailCall( + std::pair<const FunctionDecl *, SourceLocation> Global) { + MustTailCallUndefinedGlobals.insert(Global); + } + private: + bool shouldDropDLLAttribute(const Decl *D, const llvm::GlobalValue *GV) const; + llvm::Constant *GetOrCreateLLVMFunction( StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable, bool DontDefer = false, bool IsThunk = false, llvm::AttributeList ExtraAttrs = llvm::AttributeList(), ForDefinition_t IsForDefinition = NotForDefinition); + // Adds a declaration to the list of multi version functions if not present. + void AddDeferredMultiVersionResolverToEmit(GlobalDecl GD); + // References to multiversion functions are resolved through an implicitly // defined resolver function. This function is responsible for creating // the resolver symbol for the provided declaration. The value returned @@ -1619,6 +1719,7 @@ private: void EmitGlobalVarDefinition(const VarDecl *D, bool IsTentative = false); void EmitExternalVarDeclaration(const VarDecl *D); + void EmitExternalFunctionDeclaration(const FunctionDecl *D); void EmitAliasDefinition(GlobalDecl GD); void emitIFuncDefinition(GlobalDecl GD); void emitCPUDispatchDefinition(GlobalDecl GD); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp index fb4e86e8bd80..cfcdb5911b58 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp @@ -23,13 +23,15 @@ #include "llvm/Support/MD5.h" #include <optional> +namespace llvm { +extern cl::opt<bool> EnableSingleByteCoverage; +} // namespace llvm + static llvm::cl::opt<bool> EnableValueProfiling("enable-value-profiling", llvm::cl::desc("Enable value profiling"), llvm::cl::Hidden, llvm::cl::init(false)); -extern llvm::cl::opt<bool> SystemHeadersCoverage; - using namespace clang; using namespace CodeGen; @@ -163,10 +165,8 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { PGOHash Hash; /// The map of statements to counters. llvm::DenseMap<const Stmt *, unsigned> &CounterMap; - /// The next bitmap byte index to assign. - unsigned NextMCDCBitmapIdx; - /// The map of statements to MC/DC bitmap coverage objects. - llvm::DenseMap<const Stmt *, unsigned> &MCDCBitmapMap; + /// The state of MC/DC Coverage in this function. + MCDC::State &MCDCState; /// Maximum number of supported MC/DC conditions in a boolean expression. unsigned MCDCMaxCond; /// The profile version. @@ -176,11 +176,11 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { MapRegionCounters(PGOHashVersion HashVersion, uint64_t ProfileVersion, llvm::DenseMap<const Stmt *, unsigned> &CounterMap, - llvm::DenseMap<const Stmt *, unsigned> &MCDCBitmapMap, - unsigned MCDCMaxCond, DiagnosticsEngine &Diag) + MCDC::State &MCDCState, unsigned MCDCMaxCond, + DiagnosticsEngine &Diag) : NextCounter(0), Hash(HashVersion), CounterMap(CounterMap), - NextMCDCBitmapIdx(0), MCDCBitmapMap(MCDCBitmapMap), - MCDCMaxCond(MCDCMaxCond), ProfileVersion(ProfileVersion), Diag(Diag) {} + MCDCState(MCDCState), MCDCMaxCond(MCDCMaxCond), + ProfileVersion(ProfileVersion), Diag(Diag) {} // Blocks and lambdas are handled as separate functions, so we need not // traverse them in the parent context. @@ -310,11 +310,8 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { return true; } - // Otherwise, allocate the number of bytes required for the bitmap - // based on the number of conditions. Must be at least 1-byte long. - MCDCBitmapMap[BinOp] = NextMCDCBitmapIdx; - unsigned SizeInBits = std::max<unsigned>(1L << NumCond, CHAR_BIT); - NextMCDCBitmapIdx += SizeInBits / CHAR_BIT; + // Otherwise, allocate the Decision. + MCDCState.DecisionByStmt[BinOp].BitmapIdx = 0; } return true; } @@ -346,6 +343,14 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { return Base::VisitBinaryOperator(S); } + bool VisitConditionalOperator(ConditionalOperator *S) { + if (llvm::EnableSingleByteCoverage && S->getTrueExpr()) + CounterMap[S->getTrueExpr()] = NextCounter++; + if (llvm::EnableSingleByteCoverage && S->getFalseExpr()) + CounterMap[S->getFalseExpr()] = NextCounter++; + return Base::VisitConditionalOperator(S); + } + /// Include \p S in the function hash. bool VisitStmt(Stmt *S) { auto Type = updateCounterMappings(S); @@ -361,8 +366,21 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { if (Hash.getHashVersion() == PGO_HASH_V1) return Base::TraverseIfStmt(If); + // When single byte coverage mode is enabled, add a counter to then and + // else. + bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage; + for (Stmt *CS : If->children()) { + if (!CS || NoSingleByteCoverage) + continue; + if (CS == If->getThen()) + CounterMap[If->getThen()] = NextCounter++; + else if (CS == If->getElse()) + CounterMap[If->getElse()] = NextCounter++; + } + // Otherwise, keep track of which branch we're in while traversing. VisitStmt(If); + for (Stmt *CS : If->children()) { if (!CS) continue; @@ -376,6 +394,81 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { return true; } + bool TraverseWhileStmt(WhileStmt *While) { + // When single byte coverage mode is enabled, add a counter to condition and + // body. + bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage; + for (Stmt *CS : While->children()) { + if (!CS || NoSingleByteCoverage) + continue; + if (CS == While->getCond()) + CounterMap[While->getCond()] = NextCounter++; + else if (CS == While->getBody()) + CounterMap[While->getBody()] = NextCounter++; + } + + Base::TraverseWhileStmt(While); + if (Hash.getHashVersion() != PGO_HASH_V1) + Hash.combine(PGOHash::EndOfScope); + return true; + } + + bool TraverseDoStmt(DoStmt *Do) { + // When single byte coverage mode is enabled, add a counter to condition and + // body. + bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage; + for (Stmt *CS : Do->children()) { + if (!CS || NoSingleByteCoverage) + continue; + if (CS == Do->getCond()) + CounterMap[Do->getCond()] = NextCounter++; + else if (CS == Do->getBody()) + CounterMap[Do->getBody()] = NextCounter++; + } + + Base::TraverseDoStmt(Do); + if (Hash.getHashVersion() != PGO_HASH_V1) + Hash.combine(PGOHash::EndOfScope); + return true; + } + + bool TraverseForStmt(ForStmt *For) { + // When single byte coverage mode is enabled, add a counter to condition, + // increment and body. + bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage; + for (Stmt *CS : For->children()) { + if (!CS || NoSingleByteCoverage) + continue; + if (CS == For->getCond()) + CounterMap[For->getCond()] = NextCounter++; + else if (CS == For->getInc()) + CounterMap[For->getInc()] = NextCounter++; + else if (CS == For->getBody()) + CounterMap[For->getBody()] = NextCounter++; + } + + Base::TraverseForStmt(For); + if (Hash.getHashVersion() != PGO_HASH_V1) + Hash.combine(PGOHash::EndOfScope); + return true; + } + + bool TraverseCXXForRangeStmt(CXXForRangeStmt *ForRange) { + // When single byte coverage mode is enabled, add a counter to body. + bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage; + for (Stmt *CS : ForRange->children()) { + if (!CS || NoSingleByteCoverage) + continue; + if (CS == ForRange->getBody()) + CounterMap[ForRange->getBody()] = NextCounter++; + } + + Base::TraverseCXXForRangeStmt(ForRange); + if (Hash.getHashVersion() != PGO_HASH_V1) + Hash.combine(PGOHash::EndOfScope); + return true; + } + // If the statement type \p N is nestable, and its nesting impacts profile // stability, define a custom traversal which tracks the end of the statement // in the hash (provided we're not using the V1 hash). @@ -387,10 +480,6 @@ struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { return true; \ } - DEFINE_NESTABLE_TRAVERSAL(WhileStmt) - DEFINE_NESTABLE_TRAVERSAL(DoStmt) - DEFINE_NESTABLE_TRAVERSAL(ForStmt) - DEFINE_NESTABLE_TRAVERSAL(CXXForRangeStmt) DEFINE_NESTABLE_TRAVERSAL(ObjCForCollectionStmt) DEFINE_NESTABLE_TRAVERSAL(CXXTryStmt) DEFINE_NESTABLE_TRAVERSAL(CXXCatchStmt) @@ -955,13 +1044,17 @@ void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) { if (Fn->hasFnAttribute(llvm::Attribute::SkipProfile)) return; + SourceManager &SM = CGM.getContext().getSourceManager(); + if (!llvm::coverage::SystemHeadersCoverage && + SM.isInSystemHeader(D->getLocation())) + return; + setFuncName(Fn); mapRegionCounters(D); if (CGM.getCodeGenOpts().CoverageMapping) emitCounterRegionMapping(D); if (PGOReader) { - SourceManager &SM = CGM.getContext().getSourceManager(); loadRegionCounts(PGOReader, SM.isInMainFile(D->getLocation())); computeRegionCounts(D); applyFunctionAttributes(PGOReader, Fn); @@ -987,13 +1080,14 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) { // for most embedded applications. Setting a maximum value prevents the // bitmap footprint from growing too large without the user's knowledge. In // the future, this value could be adjusted with a command-line option. - unsigned MCDCMaxConditions = (CGM.getCodeGenOpts().MCDCCoverage) ? 6 : 0; + unsigned MCDCMaxConditions = + (CGM.getCodeGenOpts().MCDCCoverage ? CGM.getCodeGenOpts().MCDCMaxConds + : 0); RegionCounterMap.reset(new llvm::DenseMap<const Stmt *, unsigned>); - RegionMCDCBitmapMap.reset(new llvm::DenseMap<const Stmt *, unsigned>); + RegionMCDCState.reset(new MCDC::State); MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap, - *RegionMCDCBitmapMap, MCDCMaxConditions, - CGM.getDiags()); + *RegionMCDCState, MCDCMaxConditions, CGM.getDiags()); if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) Walker.TraverseDecl(const_cast<FunctionDecl *>(FD)); else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D)) @@ -1004,7 +1098,6 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) { Walker.TraverseDecl(const_cast<CapturedDecl *>(CD)); assert(Walker.NextCounter > 0 && "no entry counter mapped for decl"); NumRegionCounters = Walker.NextCounter; - MCDCBitmapBytes = Walker.NextMCDCBitmapIdx; FunctionHash = Walker.Hash.finalize(); } @@ -1027,7 +1120,7 @@ bool CodeGenPGO::skipRegionMappingForDecl(const Decl *D) { // Don't map the functions in system headers. const auto &SM = CGM.getContext().getSourceManager(); auto Loc = D->getBody()->getBeginLoc(); - return !SystemHeadersCoverage && SM.isInSystemHeader(Loc); + return !llvm::coverage::SystemHeadersCoverage && SM.isInSystemHeader(Loc); } void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { @@ -1036,11 +1129,10 @@ void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { std::string CoverageMapping; llvm::raw_string_ostream OS(CoverageMapping); - RegionCondIDMap.reset(new llvm::DenseMap<const Stmt *, unsigned>); + RegionMCDCState->BranchByStmt.clear(); CoverageMappingGen MappingGen( *CGM.getCoverageMapping(), CGM.getContext().getSourceManager(), - CGM.getLangOpts(), RegionCounterMap.get(), RegionMCDCBitmapMap.get(), - RegionCondIDMap.get()); + CGM.getLangOpts(), RegionCounterMap.get(), RegionMCDCState.get()); MappingGen.emitCounterMapping(D, OS); OS.flush(); @@ -1096,8 +1188,8 @@ CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, Fn->setEntryCount(FunctionCount); } -void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, - llvm::Value *StepV) { +void CodeGenPGO::emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, + llvm::Value *StepV) { if (!RegionCounterMap || !Builder.GetInsertBlock()) return; @@ -1107,13 +1199,18 @@ void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, Builder.getInt64(FunctionHash), Builder.getInt32(NumRegionCounters), Builder.getInt32(Counter), StepV}; - if (!StepV) - Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), + + if (llvm::EnableSingleByteCoverage) + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_cover), ArrayRef(Args, 4)); - else - Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment_step), - ArrayRef(Args)); + else { + if (!StepV) + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), + ArrayRef(Args, 4)); + else + Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment_step), Args); + } } bool CodeGenPGO::canEmitMCDCCoverage(const CGBuilderTy &Builder) { @@ -1122,7 +1219,7 @@ bool CodeGenPGO::canEmitMCDCCoverage(const CGBuilderTy &Builder) { } void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) { - if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap) + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext()); @@ -1132,25 +1229,31 @@ void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) { // anything. llvm::Value *Args[3] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), - Builder.getInt32(MCDCBitmapBytes)}; + Builder.getInt32(RegionMCDCState->BitmapBits)}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_parameters), Args); } void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr) { - if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap) + Address MCDCCondBitmapAddr, + CodeGenFunction &CGF) { + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; S = S->IgnoreParens(); - auto ExprMCDCBitmapMapIterator = RegionMCDCBitmapMap->find(S); - if (ExprMCDCBitmapMapIterator == RegionMCDCBitmapMap->end()) + auto DecisionStateIter = RegionMCDCState->DecisionByStmt.find(S); + if (DecisionStateIter == RegionMCDCState->DecisionByStmt.end()) return; - // Extract the ID of the global bitmap associated with this expression. - unsigned MCDCTestVectorBitmapID = ExprMCDCBitmapMapIterator->second; + // Don't create tvbitmap_update if the record is allocated but excluded. + // Or `bitmap |= (1 << 0)` would be wrongly executed to the next bitmap. + if (DecisionStateIter->second.Indices.size() == 0) + return; + + // Extract the offset of the global bitmap associated with this expression. + unsigned MCDCTestVectorBitmapOffset = DecisionStateIter->second.BitmapIdx; auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext()); // Emit intrinsic responsible for updating the global bitmap corresponding to @@ -1158,23 +1261,22 @@ void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, // from a pointer to a dedicated temporary value on the stack that is itself // updated via emitMCDCCondBitmapReset() and emitMCDCCondBitmapUpdate(). The // index represents an executed test vector. - llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + llvm::Value *Args[4] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), - Builder.getInt32(MCDCBitmapBytes), - Builder.getInt32(MCDCTestVectorBitmapID), - MCDCCondBitmapAddr.getPointer()}; + Builder.getInt32(MCDCTestVectorBitmapOffset), + MCDCCondBitmapAddr.emitRawPointer(CGF)}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args); } void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr) { - if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap) + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; S = S->IgnoreParens(); - if (RegionMCDCBitmapMap->find(S) == RegionMCDCBitmapMap->end()) + if (!RegionMCDCState->DecisionByStmt.contains(S)) return; // Emit intrinsic that resets a dedicated temporary value on the stack to 0. @@ -1183,8 +1285,9 @@ void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr, - llvm::Value *Val) { - if (!canEmitMCDCCoverage(Builder) || !RegionCondIDMap) + llvm::Value *Val, + CodeGenFunction &CGF) { + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; // Even though, for simplicity, parentheses and unary logical-NOT operators @@ -1196,26 +1299,29 @@ void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, // also make debugging a bit easier. S = CodeGenFunction::stripCond(S); - auto ExprMCDCConditionIDMapIterator = RegionCondIDMap->find(S); - if (ExprMCDCConditionIDMapIterator == RegionCondIDMap->end()) + auto BranchStateIter = RegionMCDCState->BranchByStmt.find(S); + if (BranchStateIter == RegionMCDCState->BranchByStmt.end()) return; // Extract the ID of the condition we are setting in the bitmap. - unsigned CondID = ExprMCDCConditionIDMapIterator->second; - assert(CondID > 0 && "Condition has no ID!"); + const auto &Branch = BranchStateIter->second; + assert(Branch.ID >= 0 && "Condition has no ID!"); + assert(Branch.DecisionStmt); + + // Cancel the emission if the Decision is erased after the allocation. + const auto DecisionIter = + RegionMCDCState->DecisionByStmt.find(Branch.DecisionStmt); + if (DecisionIter == RegionMCDCState->DecisionByStmt.end()) + return; - auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext()); + const auto &TVIdxs = DecisionIter->second.Indices[Branch.ID]; - // Emit intrinsic that updates a dedicated temporary value on the stack after - // a condition is evaluated. After the set of conditions has been updated, - // the resulting value is used to update the boolean expression's bitmap. - llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), - Builder.getInt64(FunctionHash), - Builder.getInt32(CondID - 1), - MCDCCondBitmapAddr.getPointer(), Val}; - Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update), - Args); + auto *CurTV = Builder.CreateLoad(MCDCCondBitmapAddr, + "mcdc." + Twine(Branch.ID + 1) + ".cur"); + auto *NewTV = Builder.CreateAdd(CurTV, Builder.getInt32(TVIdxs[true])); + NewTV = Builder.CreateSelect( + Val, NewTV, Builder.CreateAdd(CurTV, Builder.getInt32(TVIdxs[false]))); + Builder.CreateStore(NewTV, MCDCCondBitmapAddr); } void CodeGenPGO::setValueProfilingFlag(llvm::Module &M) { @@ -1224,6 +1330,30 @@ void CodeGenPGO::setValueProfilingFlag(llvm::Module &M) { uint32_t(EnableValueProfiling)); } +void CodeGenPGO::setProfileVersion(llvm::Module &M) { + if (CGM.getCodeGenOpts().hasProfileClangInstr() && + llvm::EnableSingleByteCoverage) { + const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); + llvm::Type *IntTy64 = llvm::Type::getInt64Ty(M.getContext()); + uint64_t ProfileVersion = + (INSTR_PROF_RAW_VERSION | VARIANT_MASK_BYTE_COVERAGE); + + auto IRLevelVersionVariable = new llvm::GlobalVariable( + M, IntTy64, true, llvm::GlobalValue::WeakAnyLinkage, + llvm::Constant::getIntegerValue(IntTy64, + llvm::APInt(64, ProfileVersion)), + VarName); + + IRLevelVersionVariable->setVisibility(llvm::GlobalValue::HiddenVisibility); + llvm::Triple TT(M.getTargetTriple()); + if (TT.supportsCOMDAT()) { + IRLevelVersionVariable->setLinkage(llvm::GlobalValue::ExternalLinkage); + IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName)); + } + IRLevelVersionVariable->setDSOLocal(true); + } +} + // This method either inserts a call to the profile run-time during // instrumentation or puts profile data into metadata for PGO use. void CodeGenPGO::valueProfile(CGBuilderTy &Builder, uint32_t ValueKind, diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.h index 6596b6c35277..9d66ffad6f43 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenPGO.h @@ -16,6 +16,7 @@ #include "CGBuilder.h" #include "CodeGenModule.h" #include "CodeGenTypes.h" +#include "MCDCState.h" #include "llvm/ProfileData/InstrProfReader.h" #include <array> #include <memory> @@ -33,21 +34,18 @@ private: std::array <unsigned, llvm::IPVK_Last + 1> NumValueSites; unsigned NumRegionCounters; - unsigned MCDCBitmapBytes; uint64_t FunctionHash; std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap; - std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionMCDCBitmapMap; - std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCondIDMap; std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap; std::unique_ptr<llvm::InstrProfRecord> ProfRecord; + std::unique_ptr<MCDC::State> RegionMCDCState; std::vector<uint64_t> RegionCounts; uint64_t CurrentRegionCount; public: CodeGenPGO(CodeGenModule &CGModule) : CGM(CGModule), FuncNameVar(nullptr), NumValueSites({{0}}), - NumRegionCounters(0), MCDCBitmapBytes(0), FunctionHash(0), - CurrentRegionCount(0) {} + NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0) {} /// Whether or not we have PGO region data for the current function. This is /// false both when we have no data at all and when our data has been @@ -96,6 +94,8 @@ public: // Set a module flag indicating if value profiling is enabled. void setValueProfilingFlag(llvm::Module &M); + void setProfileVersion(llvm::Module &M); + private: void setFuncName(llvm::Function *Fn); void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage); @@ -110,15 +110,17 @@ private: bool canEmitMCDCCoverage(const CGBuilderTy &Builder); public: - void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, - llvm::Value *StepV); + void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, + llvm::Value *StepV); void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr); + Address MCDCCondBitmapAddr, + CodeGenFunction &CGF); void emitMCDCParameters(CGBuilderTy &Builder); void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr); void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr, llvm::Value *Val); + Address MCDCCondBitmapAddr, llvm::Value *Val, + CodeGenFunction &CGF); /// Return the region count for the counter at the given index. uint64_t getRegionCount(const Stmt *S) { diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp index dc288bc3f615..2ce558d4bdf3 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp @@ -15,27 +15,32 @@ //===----------------------------------------------------------------------===// #include "CodeGenTBAA.h" +#include "ABIInfoImpl.h" +#include "CGCXXABI.h" +#include "CGRecordLayout.h" +#include "CodeGenTypes.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" #include "clang/AST/Mangle.h" #include "clang/AST/RecordLayout.h" #include "clang/Basic/CodeGenOptions.h" +#include "clang/Basic/TargetInfo.h" #include "llvm/ADT/SmallSet.h" #include "llvm/IR/Constants.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/Support/Debug.h" using namespace clang; using namespace CodeGen; -CodeGenTBAA::CodeGenTBAA(ASTContext &Ctx, llvm::Module &M, - const CodeGenOptions &CGO, - const LangOptions &Features, MangleContext &MContext) - : Context(Ctx), Module(M), CodeGenOpts(CGO), - Features(Features), MContext(MContext), MDHelper(M.getContext()), - Root(nullptr), Char(nullptr) -{} +CodeGenTBAA::CodeGenTBAA(ASTContext &Ctx, CodeGenTypes &CGTypes, + llvm::Module &M, const CodeGenOptions &CGO, + const LangOptions &Features) + : Context(Ctx), CGTypes(CGTypes), Module(M), CodeGenOpts(CGO), + Features(Features), MDHelper(M.getContext()), Root(nullptr), + Char(nullptr) {} CodeGenTBAA::~CodeGenTBAA() { } @@ -95,8 +100,6 @@ static bool TypeHasMayAlias(QualType QTy) { /// Check if the given type is a valid base type to be used in access tags. static bool isValidBaseType(QualType QTy) { - if (QTy->isReferenceType()) - return false; if (const RecordType *TTy = QTy->getAs<RecordType>()) { const RecordDecl *RD = TTy->getDecl()->getDefinition(); // Incomplete types are not valid base access types. @@ -184,10 +187,56 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) { return getChar(); // Handle pointers and references. - // TODO: Implement C++'s type "similarity" and consider dis-"similar" - // pointers distinct. - if (Ty->isPointerType() || Ty->isReferenceType()) - return createScalarTypeNode("any pointer", getChar(), Size); + // + // C has a very strict rule for pointer aliasing. C23 6.7.6.1p2: + // For two pointer types to be compatible, both shall be identically + // qualified and both shall be pointers to compatible types. + // + // This rule is impractically strict; we want to at least ignore CVR + // qualifiers. Distinguishing by CVR qualifiers would make it UB to + // e.g. cast a `char **` to `const char * const *` and dereference it, + // which is too common and useful to invalidate. C++'s similar types + // rule permits qualifier differences in these nested positions; in fact, + // C++ even allows that cast as an implicit conversion. + // + // Other qualifiers could theoretically be distinguished, especially if + // they involve a significant representation difference. We don't + // currently do so, however. + // + // Computing the pointee type string recursively is implicitly more + // forgiving than the standards require. Effectively, we are turning + // the question "are these types compatible/similar" into "are + // accesses to these types allowed to alias". In both C and C++, + // the latter question has special carve-outs for signedness + // mismatches that only apply at the top level. As a result, we are + // allowing e.g. `int *` l-values to access `unsigned *` objects. + if (Ty->isPointerType() || Ty->isReferenceType()) { + llvm::MDNode *AnyPtr = createScalarTypeNode("any pointer", getChar(), Size); + if (!CodeGenOpts.PointerTBAA) + return AnyPtr; + // Compute the depth of the pointer and generate a tag of the form "p<depth> + // <base type tag>". + unsigned PtrDepth = 0; + do { + PtrDepth++; + Ty = Ty->getPointeeType().getTypePtr(); + } while (Ty->isPointerType()); + // TODO: Implement C++'s type "similarity" and consider dis-"similar" + // pointers distinct for non-builtin types. + if (isa<BuiltinType>(Ty)) { + llvm::MDNode *ScalarMD = getTypeInfoHelper(Ty); + StringRef Name = + cast<llvm::MDString>( + ScalarMD->getOperand(CodeGenOpts.NewStructPathTBAA ? 2 : 0)) + ->getString(); + SmallString<256> OutName("p"); + OutName += std::to_string(PtrDepth); + OutName += " "; + OutName += Name; + return createScalarTypeNode(OutName, AnyPtr, Size); + } + return AnyPtr; + } // Accesses to arrays are accesses to objects of their element types. if (CodeGenOpts.NewStructPathTBAA && Ty->isArrayType()) @@ -208,7 +257,8 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) { SmallString<256> OutName; llvm::raw_svector_ostream Out(OutName); - MContext.mangleCanonicalTypeName(QualType(ETy, 0), Out); + CGTypes.getCXXABI().getMangleContext().mangleCanonicalTypeName( + QualType(ETy, 0), Out); return createScalarTypeNode(OutName, getChar(), Size); } @@ -240,9 +290,10 @@ llvm::MDNode *CodeGenTBAA::getTypeInfo(QualType QTy) { // aggregate will result into the may-alias access descriptor, meaning all // subsequent accesses to direct and indirect members of that aggregate will // be considered may-alias too. - // TODO: Combine getTypeInfo() and getBaseTypeInfo() into a single function. + // TODO: Combine getTypeInfo() and getValidBaseTypeInfo() into a single + // function. if (isValidBaseType(QTy)) - return getBaseTypeInfo(QTy); + return getValidBaseTypeInfo(QTy); const Type *Ty = Context.getCanonicalType(QTy).getTypePtr(); if (llvm::MDNode *N = MetadataCache[Ty]) @@ -284,6 +335,14 @@ CodeGenTBAA::CollectFields(uint64_t BaseOffset, /* Things not handled yet include: C++ base classes, bitfields, */ if (const RecordType *TTy = QTy->getAs<RecordType>()) { + if (TTy->isUnionType()) { + uint64_t Size = Context.getTypeSizeInChars(QTy).getQuantity(); + llvm::MDNode *TBAAType = getChar(); + llvm::MDNode *TBAATag = getAccessTagInfo(TBAAAccessInfo(TBAAType, Size)); + Fields.push_back( + llvm::MDBuilder::TBAAStructField(BaseOffset, Size, TBAATag)); + return true; + } const RecordDecl *RD = TTy->getDecl()->getDefinition(); if (RD->hasFlexibleArrayMember()) return false; @@ -294,14 +353,40 @@ CodeGenTBAA::CollectFields(uint64_t BaseOffset, return false; const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); + const CGRecordLayout &CGRL = CGTypes.getCGRecordLayout(RD); unsigned idx = 0; - for (RecordDecl::field_iterator i = RD->field_begin(), - e = RD->field_end(); i != e; ++i, ++idx) { - if ((*i)->isZeroSize(Context) || (*i)->isUnnamedBitfield()) + for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end(); + i != e; ++i, ++idx) { + if (isEmptyFieldForLayout(Context, *i)) + continue; + + uint64_t Offset = + BaseOffset + Layout.getFieldOffset(idx) / Context.getCharWidth(); + + // Create a single field for consecutive named bitfields using char as + // base type. + if ((*i)->isBitField()) { + const CGBitFieldInfo &Info = CGRL.getBitFieldInfo(*i); + // For big endian targets the first bitfield in the consecutive run is + // at the most-significant end; see CGRecordLowering::setBitFieldInfo + // for more information. + bool IsBE = Context.getTargetInfo().isBigEndian(); + bool IsFirst = IsBE ? Info.StorageSize - (Info.Offset + Info.Size) == 0 + : Info.Offset == 0; + if (!IsFirst) + continue; + unsigned CurrentBitFieldSize = Info.StorageSize; + uint64_t Size = + llvm::divideCeil(CurrentBitFieldSize, Context.getCharWidth()); + llvm::MDNode *TBAAType = getChar(); + llvm::MDNode *TBAATag = + getAccessTagInfo(TBAAAccessInfo(TBAAType, Size)); + Fields.push_back( + llvm::MDBuilder::TBAAStructField(Offset, Size, TBAATag)); continue; - uint64_t Offset = BaseOffset + - Layout.getFieldOffset(idx) / Context.getCharWidth(); + } + QualType FieldQTy = i->getType(); if (!CollectFields(Offset, FieldQTy, Fields, MayAlias || TypeHasMayAlias(FieldQTy))) @@ -321,6 +406,9 @@ CodeGenTBAA::CollectFields(uint64_t BaseOffset, llvm::MDNode * CodeGenTBAA::getTBAAStructInfo(QualType QTy) { + if (CodeGenOpts.OptimizationLevel == 0 || CodeGenOpts.RelaxedAliasing) + return nullptr; + const Type *Ty = Context.getCanonicalType(QTy).getTypePtr(); if (llvm::MDNode *N = StructMetadataCache[Ty]) @@ -354,7 +442,7 @@ llvm::MDNode *CodeGenTBAA::getBaseTypeInfoHelper(const Type *Ty) { if (BaseRD->isEmpty()) continue; llvm::MDNode *TypeNode = isValidBaseType(BaseQTy) - ? getBaseTypeInfo(BaseQTy) + ? getValidBaseTypeInfo(BaseQTy) : getTypeInfo(BaseQTy); if (!TypeNode) return nullptr; @@ -375,11 +463,12 @@ llvm::MDNode *CodeGenTBAA::getBaseTypeInfoHelper(const Type *Ty) { }); } for (FieldDecl *Field : RD->fields()) { - if (Field->isZeroSize(Context) || Field->isUnnamedBitfield()) + if (Field->isZeroSize(Context) || Field->isUnnamedBitField()) continue; QualType FieldQTy = Field->getType(); - llvm::MDNode *TypeNode = isValidBaseType(FieldQTy) ? - getBaseTypeInfo(FieldQTy) : getTypeInfo(FieldQTy); + llvm::MDNode *TypeNode = isValidBaseType(FieldQTy) + ? getValidBaseTypeInfo(FieldQTy) + : getTypeInfo(FieldQTy); if (!TypeNode) return nullptr; @@ -394,7 +483,8 @@ llvm::MDNode *CodeGenTBAA::getBaseTypeInfoHelper(const Type *Ty) { if (Features.CPlusPlus) { // Don't use the mangler for C code. llvm::raw_svector_ostream Out(OutName); - MContext.mangleCanonicalTypeName(QualType(Ty, 0), Out); + CGTypes.getCXXABI().getMangleContext().mangleCanonicalTypeName( + QualType(Ty, 0), Out); } else { OutName = RD->getName(); } @@ -416,9 +506,8 @@ llvm::MDNode *CodeGenTBAA::getBaseTypeInfoHelper(const Type *Ty) { return nullptr; } -llvm::MDNode *CodeGenTBAA::getBaseTypeInfo(QualType QTy) { - if (!isValidBaseType(QTy)) - return nullptr; +llvm::MDNode *CodeGenTBAA::getValidBaseTypeInfo(QualType QTy) { + assert(isValidBaseType(QTy) && "Must be a valid base type"); const Type *Ty = Context.getCanonicalType(QTy).getTypePtr(); @@ -437,6 +526,10 @@ llvm::MDNode *CodeGenTBAA::getBaseTypeInfo(QualType QTy) { return TypeNode; } +llvm::MDNode *CodeGenTBAA::getBaseTypeInfo(QualType QTy) { + return isValidBaseType(QTy) ? getValidBaseTypeInfo(QTy) : nullptr; +} + llvm::MDNode *CodeGenTBAA::getAccessTagInfo(TBAAAccessInfo Info) { assert(!Info.isIncomplete() && "Access to an object of an incomplete type!"); diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.h index a65963596fe9..ba74a39a4d25 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTBAA.h @@ -24,11 +24,11 @@ namespace clang { class ASTContext; class CodeGenOptions; class LangOptions; - class MangleContext; class QualType; class Type; namespace CodeGen { +class CodeGenTypes; // TBAAAccessKind - A kind of TBAA memory access descriptor. enum class TBAAAccessKind : unsigned { @@ -115,10 +115,10 @@ struct TBAAAccessInfo { /// while lowering AST types to LLVM types. class CodeGenTBAA { ASTContext &Context; + CodeGenTypes &CGTypes; llvm::Module &Module; const CodeGenOptions &CodeGenOpts; const LangOptions &Features; - MangleContext &MContext; // MDHelper - Helper for creating metadata. llvm::MDBuilder MDHelper; @@ -166,9 +166,13 @@ class CodeGenTBAA { /// used to describe accesses to objects of the given base type. llvm::MDNode *getBaseTypeInfoHelper(const Type *Ty); + /// getValidBaseTypeInfo - Return metadata that describes the given base + /// access type. The type must be suitable. + llvm::MDNode *getValidBaseTypeInfo(QualType QTy); + public: - CodeGenTBAA(ASTContext &Ctx, llvm::Module &M, const CodeGenOptions &CGO, - const LangOptions &Features, MangleContext &MContext); + CodeGenTBAA(ASTContext &Ctx, CodeGenTypes &CGTypes, llvm::Module &M, + const CodeGenOptions &CGO, const LangOptions &Features); ~CodeGenTBAA(); /// getTypeInfo - Get metadata used to describe accesses to objects of the @@ -187,8 +191,9 @@ public: /// the given type. llvm::MDNode *getTBAAStructInfo(QualType QTy); - /// getBaseTypeInfo - Get metadata that describes the given base access type. - /// Return null if the type is not suitable for use in TBAA access tags. + /// getBaseTypeInfo - Get metadata that describes the given base access + /// type. Return null if the type is not suitable for use in TBAA access + /// tags. llvm::MDNode *getBaseTypeInfo(QualType QTy); /// getAccessTagInfo - Get TBAA tag for a given memory access. diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypeCache.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypeCache.h index 083d69214fb3..e273ebe3b060 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypeCache.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypeCache.h @@ -51,7 +51,7 @@ struct CodeGenTypeCache { llvm::IntegerType *PtrDiffTy; }; - /// void*, void** in address space 0 + /// void*, void** in the target's default address space (often 0) union { llvm::PointerType *UnqualPtrTy; llvm::PointerType *VoidPtrTy; diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp index a6b51bfef876..f5deccdc1ba7 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp @@ -30,9 +30,8 @@ using namespace clang; using namespace CodeGen; CodeGenTypes::CodeGenTypes(CodeGenModule &cgm) - : CGM(cgm), Context(cgm.getContext()), TheModule(cgm.getModule()), - Target(cgm.getTarget()), TheCXXABI(cgm.getCXXABI()), - TheABIInfo(cgm.getTargetCodeGenInfo().getABIInfo()) { + : CGM(cgm), Context(cgm.getContext()), TheModule(cgm.getModule()), + Target(cgm.getTarget()) { SkippedLayout = false; LongDoubleReferenced = false; } @@ -43,6 +42,8 @@ CodeGenTypes::~CodeGenTypes() { delete &*I++; } +CGCXXABI &CodeGenTypes::getCXXABI() const { return getCGM().getCXXABI(); } + const CodeGenOptions &CodeGenTypes::getCodeGenOpts() const { return CGM.getCodeGenOpts(); } @@ -89,7 +90,14 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD, /// ConvertType in that it is used to convert to the memory representation for /// a type. For example, the scalar representation for _Bool is i1, but the /// memory representation is usually i8 or i32, depending on the target. -llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T, bool ForBitField) { +/// +/// We generally assume that the alloc size of this type under the LLVM +/// data layout is the same as the size of the AST type. The alignment +/// does not have to match: Clang should always use explicit alignments +/// and packed structs as necessary to produce the layout it needs. +/// But the size does need to be exactly right or else things like struct +/// layout will break. +llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) { if (T->isConstantMatrixType()) { const Type *Ty = Context.getCanonicalType(T).getTypePtr(); const ConstantMatrixType *MT = cast<ConstantMatrixType>(Ty); @@ -107,10 +115,28 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T, bool ForBitField) { return llvm::IntegerType::get(FixedVT->getContext(), BytePadded); } - // If this is a bool type, or a bit-precise integer type in a bitfield - // representation, map this integer to the target-specified size. - if ((ForBitField && T->isBitIntType()) || - (!T->isBitIntType() && R->isIntegerTy(1))) + // If T is _Bool or a _BitInt type, ConvertType will produce an IR type + // with the exact semantic bit-width of the AST type; for example, + // _BitInt(17) will turn into i17. In memory, however, we need to store + // such values extended to their full storage size as decided by AST + // layout; this is an ABI requirement. Ideally, we would always use an + // integer type that's just the bit-size of the AST type; for example, if + // sizeof(_BitInt(17)) == 4, _BitInt(17) would turn into i32. That is what's + // returned by convertTypeForLoadStore. However, that type does not + // always satisfy the size requirement on memory representation types + // describe above. For example, a 32-bit platform might reasonably set + // sizeof(_BitInt(65)) == 12, but i96 is likely to have to have an alloc size + // of 16 bytes in the LLVM data layout. In these cases, we simply return + // a byte array of the appropriate size. + if (T->isBitIntType()) { + if (typeRequiresSplitIntoByteArray(T, R)) + return llvm::ArrayType::get(CGM.Int8Ty, + Context.getTypeSizeInChars(T).getQuantity()); + return llvm::IntegerType::get(getLLVMContext(), + (unsigned)Context.getTypeSize(T)); + } + + if (R->isIntegerTy(1)) return llvm::IntegerType::get(getLLVMContext(), (unsigned)Context.getTypeSize(T)); @@ -118,6 +144,36 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T, bool ForBitField) { return R; } +bool CodeGenTypes::typeRequiresSplitIntoByteArray(QualType ASTTy, + llvm::Type *LLVMTy) { + if (!LLVMTy) + LLVMTy = ConvertType(ASTTy); + + CharUnits ASTSize = Context.getTypeSizeInChars(ASTTy); + CharUnits LLVMSize = + CharUnits::fromQuantity(getDataLayout().getTypeAllocSize(LLVMTy)); + return ASTSize != LLVMSize; +} + +llvm::Type *CodeGenTypes::convertTypeForLoadStore(QualType T, + llvm::Type *LLVMTy) { + if (!LLVMTy) + LLVMTy = ConvertType(T); + + if (T->isBitIntType()) + return llvm::Type::getIntNTy( + getLLVMContext(), Context.getTypeSizeInChars(T).getQuantity() * 8); + + if (LLVMTy->isIntegerTy(1)) + return llvm::IntegerType::get(getLLVMContext(), + (unsigned)Context.getTypeSize(T)); + + if (T->isExtVectorBoolType()) + return ConvertTypeForMem(T); + + return LLVMTy; +} + /// isRecordLayoutComplete - Return true if the specified type is already /// completely laid out. bool CodeGenTypes::isRecordLayoutComplete(const Type *Ty) const { @@ -409,7 +465,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { break; case BuiltinType::LongDouble: LongDoubleReferenced = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case BuiltinType::BFloat16: case BuiltinType::Float: case BuiltinType::Double: @@ -523,8 +579,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { return llvm::StructType::get(getLLVMContext(), EltTys); } return llvm::ScalableVectorType::get(ConvertType(Info.ElementType), - Info.EC.getKnownMinValue() * - Info.NumVectors); + Info.EC.getKnownMinValue()); } #define WASM_REF_TYPE(Name, MangledName, Id, SingletonId, AS) \ case BuiltinType::Id: { \ @@ -534,6 +589,11 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { llvm_unreachable("Unexpected wasm reference builtin type!"); \ } break; #include "clang/Basic/WebAssemblyReferenceTypes.def" +#define AMDGPU_OPAQUE_PTR_TYPE(Name, MangledName, AS, Width, Align, Id, \ + SingletonId) \ + case BuiltinType::Id: \ + return llvm::PointerType::get(getLLVMContext(), AS); +#include "clang/Basic/AMDGPUTypes.def" case BuiltinType::Dependent: #define BUILTIN_TYPE(Id, SingletonId) #define PLACEHOLDER_TYPE(Id, SingletonId) \ @@ -590,6 +650,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { ResultType = llvm::ArrayType::get(ResultType, 0); break; } + case Type::ArrayParameter: case Type::ConstantArray: { const ConstantArrayType *A = cast<ConstantArrayType>(Ty); llvm::Type *EltTy = ConvertTypeForMem(A->getElementType()); @@ -601,7 +662,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { EltTy = llvm::Type::getInt8Ty(getLLVMContext()); } - ResultType = llvm::ArrayType::get(EltTy, A->getSize().getZExtValue()); + ResultType = llvm::ArrayType::get(EltTy, A->getZExtSize()); break; } case Type::ExtVector: diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.h index 01c0c673795c..5aebf9a21223 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenTypes.h @@ -57,11 +57,6 @@ class CodeGenTypes { ASTContext &Context; llvm::Module &TheModule; const TargetInfo &Target; - CGCXXABI &TheCXXABI; - - // This should not be moved earlier, since its initialization depends on some - // of the previous reference members being already initialized - const ABIInfo &TheABIInfo; /// The opaque type map for Objective-C interfaces. All direct /// manipulation is done by the runtime interfaces, which are @@ -106,9 +101,8 @@ public: } CodeGenModule &getCGM() const { return CGM; } ASTContext &getContext() const { return Context; } - const ABIInfo &getABIInfo() const { return TheABIInfo; } const TargetInfo &getTarget() const { return Target; } - CGCXXABI &getCXXABI() const { return TheCXXABI; } + CGCXXABI &getCXXABI() const; llvm::LLVMContext &getLLVMContext() { return TheModule.getContext(); } const CodeGenOptions &getCodeGenOpts() const; @@ -126,7 +120,30 @@ public: /// ConvertType in that it is used to convert to the memory representation for /// a type. For example, the scalar representation for _Bool is i1, but the /// memory representation is usually i8 or i32, depending on the target. - llvm::Type *ConvertTypeForMem(QualType T, bool ForBitField = false); + llvm::Type *ConvertTypeForMem(QualType T); + + /// Check whether the given type needs to be laid out in memory + /// using an opaque byte-array type because its load/store type + /// does not have the correct alloc size in the LLVM data layout. + /// If this is false, the load/store type (convertTypeForLoadStore) + /// and memory representation type (ConvertTypeForMem) will + /// be the same type. + bool typeRequiresSplitIntoByteArray(QualType ASTTy, + llvm::Type *LLVMTy = nullptr); + + /// Given that T is a scalar type, return the IR type that should + /// be used for load and store operations. For example, this might + /// be i8 for _Bool or i96 for _BitInt(65). The store size of the + /// load/store type (as reported by LLVM's data layout) is always + /// the same as the alloc size of the memory representation type + /// returned by ConvertTypeForMem. + /// + /// As an optimization, if you already know the scalar value type + /// for T (as would be returned by ConvertType), you can pass + /// it as the second argument so that it does not need to be + /// recomputed in common cases where the value type and + /// load/store type are the same. + llvm::Type *convertTypeForLoadStore(QualType T, llvm::Type *LLVMTy = nullptr); /// GetFunctionType - Get the LLVM function type for \arg Info. llvm::FunctionType *GetFunctionType(const CGFunctionInfo &Info); diff --git a/contrib/llvm-project/clang/lib/CodeGen/ConstantEmitter.h b/contrib/llvm-project/clang/lib/CodeGen/ConstantEmitter.h index a55da0dcad79..581b05ae87ad 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ConstantEmitter.h +++ b/contrib/llvm-project/clang/lib/CodeGen/ConstantEmitter.h @@ -103,8 +103,9 @@ public: /// expression is known to be a constant expression with either a fairly /// simple type or a known simple form. llvm::Constant *emitAbstract(const Expr *E, QualType T); - llvm::Constant *emitAbstract(SourceLocation loc, const APValue &value, - QualType T); + llvm::Constant * + emitAbstract(SourceLocation loc, const APValue &value, QualType T, + bool EnablePtrAuthFunctionTypeDiscrimination = true); /// Try to emit the result of the given expression as an abstract constant. llvm::Constant *tryEmitAbstract(const Expr *E, QualType T); @@ -113,6 +114,9 @@ public: llvm::Constant *tryEmitAbstract(const APValue &value, QualType T); llvm::Constant *tryEmitAbstractForMemory(const APValue &value, QualType T); + llvm::Constant *tryEmitConstantSignedPointer(llvm::Constant *Ptr, + PointerAuthQualifier Auth); + llvm::Constant *tryEmitConstantExpr(const ConstantExpr *CE); llvm::Constant *emitNullForMemory(QualType T) { @@ -135,7 +139,9 @@ public: llvm::Constant *tryEmitPrivate(const Expr *E, QualType T); llvm::Constant *tryEmitPrivateForMemory(const Expr *E, QualType T); - llvm::Constant *tryEmitPrivate(const APValue &value, QualType T); + llvm::Constant * + tryEmitPrivate(const APValue &value, QualType T, + bool EnablePtrAuthFunctionTypeDiscrimination = true); llvm::Constant *tryEmitPrivateForMemory(const APValue &value, QualType T); /// Get the address of the current location. This is a constant diff --git a/contrib/llvm-project/clang/lib/CodeGen/ConstantInitBuilder.cpp b/contrib/llvm-project/clang/lib/CodeGen/ConstantInitBuilder.cpp index 3cf69f3b6415..549d5dd66b12 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ConstantInitBuilder.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/ConstantInitBuilder.cpp @@ -296,3 +296,21 @@ ConstantAggregateBuilderBase::finishStruct(llvm::StructType *ty) { buffer.erase(buffer.begin() + Begin, buffer.end()); return constant; } + +/// Sign the given pointer and add it to the constant initializer +/// currently being built. +void ConstantAggregateBuilderBase::addSignedPointer( + llvm::Constant *Pointer, const PointerAuthSchema &Schema, + GlobalDecl CalleeDecl, QualType CalleeType) { + if (!Schema || !Builder.CGM.shouldSignPointer(Schema)) + return add(Pointer); + + llvm::Constant *StorageAddress = nullptr; + if (Schema.isAddressDiscriminated()) { + StorageAddress = getAddrOfCurrentPosition(Pointer->getType()); + } + + llvm::Constant *SignedPointer = Builder.CGM.getConstantSignedPointer( + Pointer, Schema, StorageAddress, CalleeDecl, CalleeType); + add(SignedPointer); +} diff --git a/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp b/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp index 0c43317642bc..67a9caf8b4ec 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -17,6 +17,7 @@ #include "clang/Basic/FileManager.h" #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Lex/Lexer.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" @@ -31,16 +32,26 @@ // is textually included. #define COVMAP_V3 +namespace llvm { +cl::opt<bool> + EnableSingleByteCoverage("enable-single-byte-coverage", + llvm::cl::ZeroOrMore, + llvm::cl::desc("Enable single byte coverage"), + llvm::cl::Hidden, llvm::cl::init(false)); +} // namespace llvm + static llvm::cl::opt<bool> EmptyLineCommentCoverage( "emptyline-comment-coverage", llvm::cl::desc("Emit emptylines and comment lines as skipped regions (only " "disable it on test)"), llvm::cl::init(true), llvm::cl::Hidden); -llvm::cl::opt<bool> SystemHeadersCoverage( +namespace llvm::coverage { +cl::opt<bool> SystemHeadersCoverage( "system-headers-coverage", - llvm::cl::desc("Enable collecting coverage from system headers"), - llvm::cl::init(false), llvm::cl::Hidden); + cl::desc("Enable collecting coverage from system headers"), cl::init(false), + cl::Hidden); +} using namespace clang; using namespace CodeGen; @@ -95,9 +106,6 @@ void CoverageSourceInfo::updateNextTokLoc(SourceLocation Loc) { } namespace { -using MCDCConditionID = CounterMappingRegion::MCDCConditionID; -using MCDCParameters = CounterMappingRegion::MCDCParameters; - /// A region of source code that can be mapped to a counter. class SourceMappingRegion { /// Primary Counter that is also used for Branch Regions for "True" branches. @@ -107,7 +115,7 @@ class SourceMappingRegion { std::optional<Counter> FalseCount; /// Parameters used for Modified Condition/Decision Coverage - MCDCParameters MCDCParams; + mcdc::Parameters MCDCParams; /// The region's starting location. std::optional<SourceLocation> LocStart; @@ -131,7 +139,7 @@ public: SkippedRegion(false) {} SourceMappingRegion(Counter Count, std::optional<Counter> FalseCount, - MCDCParameters MCDCParams, + mcdc::Parameters MCDCParams, std::optional<SourceLocation> LocStart, std::optional<SourceLocation> LocEnd, bool GapRegion = false) @@ -139,7 +147,7 @@ public: LocStart(LocStart), LocEnd(LocEnd), GapRegion(GapRegion), SkippedRegion(false) {} - SourceMappingRegion(MCDCParameters MCDCParams, + SourceMappingRegion(mcdc::Parameters MCDCParams, std::optional<SourceLocation> LocStart, std::optional<SourceLocation> LocEnd) : MCDCParams(MCDCParams), LocStart(LocStart), LocEnd(LocEnd), @@ -185,9 +193,25 @@ public: bool isBranch() const { return FalseCount.has_value(); } - bool isMCDCDecision() const { return MCDCParams.NumConditions != 0; } + bool isMCDCBranch() const { + return std::holds_alternative<mcdc::BranchParameters>(MCDCParams); + } - const MCDCParameters &getMCDCParams() const { return MCDCParams; } + const auto &getMCDCBranchParams() const { + return mcdc::getParams<const mcdc::BranchParameters>(MCDCParams); + } + + bool isMCDCDecision() const { + return std::holds_alternative<mcdc::DecisionParameters>(MCDCParams); + } + + const auto &getMCDCDecisionParams() const { + return mcdc::getParams<const mcdc::DecisionParameters>(MCDCParams); + } + + const mcdc::Parameters &getMCDCParams() const { return MCDCParams; } + + void resetMCDCParams() { MCDCParams = mcdc::Parameters(); } }; /// Spelling locations for the start and end of a source region. @@ -278,10 +302,36 @@ public: return SM.getLocForEndOfFile(SM.getFileID(Loc)); } - /// Find out where the current file is included or macro is expanded. - SourceLocation getIncludeOrExpansionLoc(SourceLocation Loc) { - return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).getBegin() - : SM.getIncludeLoc(SM.getFileID(Loc)); + /// Find out where a macro is expanded. If the immediate result is a + /// <scratch space>, keep looking until the result isn't. Return a pair of + /// \c SourceLocation. The first object is always the begin sloc of found + /// result. The second should be checked by the caller: if it has value, it's + /// the end sloc of the found result. Otherwise the while loop didn't get + /// executed, which means the location wasn't changed and the caller has to + /// learn the end sloc from somewhere else. + std::pair<SourceLocation, std::optional<SourceLocation>> + getNonScratchExpansionLoc(SourceLocation Loc) { + std::optional<SourceLocation> EndLoc = std::nullopt; + while (Loc.isMacroID() && + SM.isWrittenInScratchSpace(SM.getSpellingLoc(Loc))) { + auto ExpansionRange = SM.getImmediateExpansionRange(Loc); + Loc = ExpansionRange.getBegin(); + EndLoc = ExpansionRange.getEnd(); + } + return std::make_pair(Loc, EndLoc); + } + + /// Find out where the current file is included or macro is expanded. If + /// \c AcceptScratch is set to false, keep looking for expansions until the + /// found sloc is not a <scratch space>. + SourceLocation getIncludeOrExpansionLoc(SourceLocation Loc, + bool AcceptScratch = true) { + if (!Loc.isMacroID()) + return SM.getIncludeLoc(SM.getFileID(Loc)); + Loc = SM.getImmediateExpansionRange(Loc).getBegin(); + if (AcceptScratch) + return Loc; + return getNonScratchExpansionLoc(Loc).first; } /// Return true if \c Loc is a location in a built-in macro. @@ -325,16 +375,35 @@ public: llvm::SmallSet<FileID, 8> Visited; SmallVector<std::pair<SourceLocation, unsigned>, 8> FileLocs; - for (const auto &Region : SourceRegions) { + for (auto &Region : SourceRegions) { SourceLocation Loc = Region.getBeginLoc(); + + // Replace Region with its definition if it is in <scratch space>. + auto NonScratchExpansionLoc = getNonScratchExpansionLoc(Loc); + auto EndLoc = NonScratchExpansionLoc.second; + if (EndLoc.has_value()) { + Loc = NonScratchExpansionLoc.first; + Region.setStartLoc(Loc); + Region.setEndLoc(EndLoc.value()); + } + + // Replace Loc with FileLoc if it is expanded with system headers. + if (!SystemHeadersCoverage && SM.isInSystemMacro(Loc)) { + auto BeginLoc = SM.getSpellingLoc(Loc); + auto EndLoc = SM.getSpellingLoc(Region.getEndLoc()); + if (SM.isWrittenInSameFile(BeginLoc, EndLoc)) { + Loc = SM.getFileLoc(Loc); + Region.setStartLoc(Loc); + Region.setEndLoc(SM.getFileLoc(Region.getEndLoc())); + } + } + FileID File = SM.getFileID(Loc); if (!Visited.insert(File).second) continue; - // Do not map FileID's associated with system headers unless collecting - // coverage from system headers is explicitly enabled. - if (!SystemHeadersCoverage && SM.isInSystemHeader(SM.getSpellingLoc(Loc))) - continue; + assert(SystemHeadersCoverage || + !SM.isInSystemHeader(SM.getSpellingLoc(Loc))); unsigned Depth = 0; for (SourceLocation Parent = getIncludeOrExpansionLoc(Loc); @@ -450,13 +519,19 @@ public: // Ignore regions from system headers unless collecting coverage from // system headers is explicitly enabled. if (!SystemHeadersCoverage && - SM.isInSystemHeader(SM.getSpellingLoc(LocStart))) + SM.isInSystemHeader(SM.getSpellingLoc(LocStart))) { + assert(!Region.isMCDCBranch() && !Region.isMCDCDecision() && + "Don't suppress the condition in system headers"); continue; + } auto CovFileID = getCoverageFileID(LocStart); // Ignore regions that don't have a file, such as builtin macros. - if (!CovFileID) + if (!CovFileID) { + assert(!Region.isMCDCBranch() && !Region.isMCDCDecision() && + "Don't suppress the condition in non-file regions"); continue; + } SourceLocation LocEnd = Region.getEndLoc(); assert(SM.isWrittenInSameFile(LocStart, LocEnd) && @@ -466,8 +541,11 @@ public: // This not only suppresses redundant regions, but sometimes prevents // creating regions with wrong counters if, for example, a statement's // body ends at the end of a nested macro. - if (Filter.count(std::make_pair(LocStart, LocEnd))) + if (Filter.count(std::make_pair(LocStart, LocEnd))) { + assert(!Region.isMCDCBranch() && !Region.isMCDCDecision() && + "Don't suppress the condition"); continue; + } // Find the spelling locations for the mapping region. SpellingRegion SR{SM, LocStart, LocEnd}; @@ -483,13 +561,13 @@ public: SR.ColumnEnd)); } else if (Region.isBranch()) { MappingRegions.push_back(CounterMappingRegion::makeBranchRegion( - Region.getCounter(), Region.getFalseCounter(), - Region.getMCDCParams(), *CovFileID, SR.LineStart, SR.ColumnStart, - SR.LineEnd, SR.ColumnEnd)); + Region.getCounter(), Region.getFalseCounter(), *CovFileID, + SR.LineStart, SR.ColumnStart, SR.LineEnd, SR.ColumnEnd, + Region.getMCDCParams())); } else if (Region.isMCDCDecision()) { MappingRegions.push_back(CounterMappingRegion::makeDecisionRegion( - Region.getMCDCParams(), *CovFileID, SR.LineStart, SR.ColumnStart, - SR.LineEnd, SR.ColumnEnd)); + Region.getMCDCDecisionParams(), *CovFileID, SR.LineStart, + SR.ColumnStart, SR.LineEnd, SR.ColumnEnd)); } else { MappingRegions.push_back(CounterMappingRegion::makeRegion( Region.getCounter(), *CovFileID, SR.LineStart, SR.ColumnStart, @@ -503,7 +581,7 @@ public: SourceRegionFilter Filter; for (const auto &FM : FileIDMapping) { SourceLocation ExpandedLoc = FM.second.second; - SourceLocation ParentLoc = getIncludeOrExpansionLoc(ExpandedLoc); + SourceLocation ParentLoc = getIncludeOrExpansionLoc(ExpandedLoc, false); if (ParentLoc.isInvalid()) continue; @@ -586,11 +664,6 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder { /// creation. struct MCDCCoverageBuilder { - struct DecisionIDPair { - MCDCConditionID TrueID = 0; - MCDCConditionID FalseID = 0; - }; - /// The AST walk recursively visits nested logical-AND or logical-OR binary /// operator nodes and then visits their LHS and RHS children nodes. As this /// happens, the algorithm will assign IDs to each operator's LHS and RHS side @@ -681,14 +754,15 @@ struct MCDCCoverageBuilder { private: CodeGenModule &CGM; - llvm::SmallVector<DecisionIDPair> DecisionStack; - llvm::DenseMap<const Stmt *, MCDCConditionID> &CondIDs; - llvm::DenseMap<const Stmt *, unsigned> &MCDCBitmapMap; - MCDCConditionID NextID = 1; + llvm::SmallVector<mcdc::ConditionIDs> DecisionStack; + MCDC::State &MCDCState; + const Stmt *DecisionStmt = nullptr; + mcdc::ConditionID NextID = 0; bool NotMapped = false; - /// Represent a sentinel value of [0,0] for the bottom of DecisionStack. - static constexpr DecisionIDPair DecisionStackSentinel{0, 0}; + /// Represent a sentinel value as a pair of final decisions for the bottom + // of DecisionStack. + static constexpr mcdc::ConditionIDs DecisionStackSentinel{-1, -1}; /// Is this a logical-AND operation? bool isLAnd(const BinaryOperator *E) const { @@ -696,38 +770,37 @@ private: } public: - MCDCCoverageBuilder(CodeGenModule &CGM, - llvm::DenseMap<const Stmt *, MCDCConditionID> &CondIDMap, - llvm::DenseMap<const Stmt *, unsigned> &MCDCBitmapMap) - : CGM(CGM), DecisionStack(1, DecisionStackSentinel), CondIDs(CondIDMap), - MCDCBitmapMap(MCDCBitmapMap) {} + MCDCCoverageBuilder(CodeGenModule &CGM, MCDC::State &MCDCState) + : CGM(CGM), DecisionStack(1, DecisionStackSentinel), + MCDCState(MCDCState) {} /// Return whether the build of the control flow map is at the top-level /// (root) of a logical operator nest in a boolean expression prior to the /// assignment of condition IDs. - bool isIdle() const { return (NextID == 1 && !NotMapped); } + bool isIdle() const { return (NextID == 0 && !NotMapped); } /// Return whether any IDs have been assigned in the build of the control /// flow map, indicating that the map is being generated for this boolean /// expression. - bool isBuilding() const { return (NextID > 1); } + bool isBuilding() const { return (NextID > 0); } /// Set the given condition's ID. - void setCondID(const Expr *Cond, MCDCConditionID ID) { - CondIDs[CodeGenFunction::stripCond(Cond)] = ID; + void setCondID(const Expr *Cond, mcdc::ConditionID ID) { + MCDCState.BranchByStmt[CodeGenFunction::stripCond(Cond)] = {ID, + DecisionStmt}; } /// Return the ID of a given condition. - MCDCConditionID getCondID(const Expr *Cond) const { - auto I = CondIDs.find(CodeGenFunction::stripCond(Cond)); - if (I == CondIDs.end()) - return 0; + mcdc::ConditionID getCondID(const Expr *Cond) const { + auto I = MCDCState.BranchByStmt.find(CodeGenFunction::stripCond(Cond)); + if (I == MCDCState.BranchByStmt.end()) + return -1; else - return I->second; + return I->second.ID; } /// Return the LHS Decision ([0,0] if not set). - const DecisionIDPair &back() const { return DecisionStack.back(); } + const mcdc::ConditionIDs &back() const { return DecisionStack.back(); } /// Push the binary operator statement to track the nest level and assign IDs /// to the operator's LHS and RHS. The RHS may be a larger subtree that is @@ -737,43 +810,47 @@ public: return; // If binary expression is disqualified, don't do mapping. - if (!isBuilding() && !MCDCBitmapMap.contains(CodeGenFunction::stripCond(E))) + if (!isBuilding() && + !MCDCState.DecisionByStmt.contains(CodeGenFunction::stripCond(E))) NotMapped = true; // Don't go any further if we don't need to map condition IDs. if (NotMapped) return; - const DecisionIDPair &ParentDecision = DecisionStack.back(); + if (NextID == 0) { + DecisionStmt = E; + assert(MCDCState.DecisionByStmt.contains(E)); + } + + const mcdc::ConditionIDs &ParentDecision = DecisionStack.back(); // If the operator itself has an assigned ID, this means it represents a // larger subtree. In this case, assign that ID to its LHS node. Its RHS // will receive a new ID below. Otherwise, assign ID+1 to LHS. - if (CondIDs.contains(CodeGenFunction::stripCond(E))) + if (MCDCState.BranchByStmt.contains(CodeGenFunction::stripCond(E))) setCondID(E->getLHS(), getCondID(E)); else setCondID(E->getLHS(), NextID++); // Assign a ID+1 for the RHS. - MCDCConditionID RHSid = NextID++; + mcdc::ConditionID RHSid = NextID++; setCondID(E->getRHS(), RHSid); // Push the LHS decision IDs onto the DecisionStack. if (isLAnd(E)) - DecisionStack.push_back({RHSid, ParentDecision.FalseID}); + DecisionStack.push_back({ParentDecision[false], RHSid}); else - DecisionStack.push_back({ParentDecision.TrueID, RHSid}); + DecisionStack.push_back({RHSid, ParentDecision[true]}); } /// Pop and return the LHS Decision ([0,0] if not set). - DecisionIDPair pop() { + mcdc::ConditionIDs pop() { if (!CGM.getCodeGenOpts().MCDCCoverage || NotMapped) - return DecisionStack.front(); + return DecisionStackSentinel; assert(DecisionStack.size() > 1); - DecisionIDPair D = DecisionStack.back(); - DecisionStack.pop_back(); - return D; + return DecisionStack.pop_back_val(); } /// Return the total number of conditions and reset the state. The number of @@ -788,15 +865,15 @@ public: // Reset state if not doing mapping. if (NotMapped) { NotMapped = false; - assert(NextID == 1); + assert(NextID == 0); return 0; } // Set number of conditions and reset. - unsigned TotalConds = NextID - 1; + unsigned TotalConds = NextID; // Reset ID back to beginning. - NextID = 1; + NextID = 0; return TotalConds; } @@ -810,12 +887,15 @@ struct CounterCoverageMappingBuilder /// The map of statements to count values. llvm::DenseMap<const Stmt *, unsigned> &CounterMap; - /// The map of statements to bitmap coverage object values. - llvm::DenseMap<const Stmt *, unsigned> &MCDCBitmapMap; + MCDC::State &MCDCState; /// A stack of currently live regions. llvm::SmallVector<SourceMappingRegion> RegionStack; + /// Set if the Expr should be handled as a leaf even if it is kind of binary + /// logical ops (&&, ||). + llvm::DenseSet<const Stmt *> LeafExprSet; + /// An object to manage MCDC regions. MCDCCoverageBuilder MCDCBuilder; @@ -835,16 +915,22 @@ struct CounterCoverageMappingBuilder /// Return a counter for the subtraction of \c RHS from \c LHS Counter subtractCounters(Counter LHS, Counter RHS, bool Simplify = true) { + assert(!llvm::EnableSingleByteCoverage && + "cannot add counters when single byte coverage mode is enabled"); return Builder.subtract(LHS, RHS, Simplify); } /// Return a counter for the sum of \c LHS and \c RHS. Counter addCounters(Counter LHS, Counter RHS, bool Simplify = true) { + assert(!llvm::EnableSingleByteCoverage && + "cannot add counters when single byte coverage mode is enabled"); return Builder.add(LHS, RHS, Simplify); } Counter addCounters(Counter C1, Counter C2, Counter C3, bool Simplify = true) { + assert(!llvm::EnableSingleByteCoverage && + "cannot add counters when single byte coverage mode is enabled"); return addCounters(addCounters(C1, C2, Simplify), C3, Simplify); } @@ -855,8 +941,6 @@ struct CounterCoverageMappingBuilder return Counter::getCounter(CounterMap[S]); } - unsigned getRegionBitmap(const Stmt *S) { return MCDCBitmapMap[S]; } - /// Push a region onto the stack. /// /// Returns the index on the stack where the region was pushed. This can be @@ -865,8 +949,7 @@ struct CounterCoverageMappingBuilder std::optional<SourceLocation> StartLoc = std::nullopt, std::optional<SourceLocation> EndLoc = std::nullopt, std::optional<Counter> FalseCount = std::nullopt, - MCDCConditionID ID = 0, MCDCConditionID TrueID = 0, - MCDCConditionID FalseID = 0) { + const mcdc::Parameters &BranchParams = std::monostate()) { if (StartLoc && !FalseCount) { MostRecentLocation = *StartLoc; @@ -885,19 +968,16 @@ struct CounterCoverageMappingBuilder StartLoc = std::nullopt; if (EndLoc && EndLoc->isInvalid()) EndLoc = std::nullopt; - RegionStack.emplace_back(Count, FalseCount, - MCDCParameters{0, 0, ID, TrueID, FalseID}, - StartLoc, EndLoc); + RegionStack.emplace_back(Count, FalseCount, BranchParams, StartLoc, EndLoc); return RegionStack.size() - 1; } - size_t pushRegion(unsigned BitmapIdx, unsigned Conditions, + size_t pushRegion(const mcdc::DecisionParameters &DecisionParams, std::optional<SourceLocation> StartLoc = std::nullopt, std::optional<SourceLocation> EndLoc = std::nullopt) { - RegionStack.emplace_back(MCDCParameters{BitmapIdx, Conditions}, StartLoc, - EndLoc); + RegionStack.emplace_back(DecisionParams, StartLoc, EndLoc); return RegionStack.size() - 1; } @@ -1024,15 +1104,12 @@ struct CounterCoverageMappingBuilder return (Cond->EvaluateAsInt(Result, CVM.getCodeGenModule().getContext())); } - using MCDCDecisionIDPair = MCDCCoverageBuilder::DecisionIDPair; - /// Create a Branch Region around an instrumentable condition for coverage /// and add it to the function's SourceRegions. A branch region tracks a /// "True" counter and a "False" counter for boolean expressions that /// result in the generation of a branch. - void - createBranchRegion(const Expr *C, Counter TrueCnt, Counter FalseCnt, - const MCDCDecisionIDPair &IDPair = MCDCDecisionIDPair()) { + void createBranchRegion(const Expr *C, Counter TrueCnt, Counter FalseCnt, + const mcdc::ConditionIDs &Conds = {}) { // Check for NULL conditions. if (!C) return; @@ -1041,10 +1118,14 @@ struct CounterCoverageMappingBuilder // region onto RegionStack but immediately pop it (which adds it to the // function's SourceRegions) because it doesn't apply to any other source // code other than the Condition. - if (CodeGenFunction::isInstrumentedCondition(C)) { - MCDCConditionID ID = MCDCBuilder.getCondID(C); - MCDCConditionID TrueID = IDPair.TrueID; - MCDCConditionID FalseID = IDPair.FalseID; + // With !SystemHeadersCoverage, binary logical ops in system headers may be + // treated as instrumentable conditions. + if (CodeGenFunction::isInstrumentedCondition(C) || + LeafExprSet.count(CodeGenFunction::stripCond(C))) { + mcdc::Parameters BranchParams; + mcdc::ConditionID ID = MCDCBuilder.getCondID(C); + if (ID >= 0) + BranchParams = mcdc::BranchParameters{ID, Conds}; // If a condition can fold to true or false, the corresponding branch // will be removed. Create a region with both counters hard-coded to @@ -1054,19 +1135,20 @@ struct CounterCoverageMappingBuilder // CodeGenFunction.c always returns false, but that is very heavy-handed. if (ConditionFoldsToBool(C)) popRegions(pushRegion(Counter::getZero(), getStart(C), getEnd(C), - Counter::getZero(), ID, TrueID, FalseID)); + Counter::getZero(), BranchParams)); else // Otherwise, create a region with the True counter and False counter. - popRegions(pushRegion(TrueCnt, getStart(C), getEnd(C), FalseCnt, ID, - TrueID, FalseID)); + popRegions(pushRegion(TrueCnt, getStart(C), getEnd(C), FalseCnt, + BranchParams)); } } /// Create a Decision Region with a BitmapIdx and number of Conditions. This /// type of region "contains" branch regions, one for each of the conditions. /// The visualization tool will group everything together. - void createDecisionRegion(const Expr *C, unsigned BitmapIdx, unsigned Conds) { - popRegions(pushRegion(BitmapIdx, Conds, getStart(C), getEnd(C))); + void createDecisionRegion(const Expr *C, + const mcdc::DecisionParameters &DecisionParams) { + popRegions(pushRegion(DecisionParams, getStart(C), getEnd(C))); } /// Create a Branch Region around a SwitchCase for code coverage @@ -1149,12 +1231,9 @@ struct CounterCoverageMappingBuilder // we've seen this region. if (StartLocs.insert(Loc).second) { if (I.isBranch()) - SourceRegions.emplace_back( - I.getCounter(), I.getFalseCounter(), - MCDCParameters{0, 0, I.getMCDCParams().ID, - I.getMCDCParams().TrueID, - I.getMCDCParams().FalseID}, - Loc, getEndOfFileOrMacro(Loc), I.isBranch()); + SourceRegions.emplace_back(I.getCounter(), I.getFalseCounter(), + I.getMCDCParams(), Loc, + getEndOfFileOrMacro(Loc), I.isBranch()); else SourceRegions.emplace_back(I.getCounter(), Loc, getEndOfFileOrMacro(Loc)); @@ -1207,6 +1286,12 @@ struct CounterCoverageMappingBuilder /// Find a valid gap range between \p AfterLoc and \p BeforeLoc. std::optional<SourceRange> findGapAreaBetween(SourceLocation AfterLoc, SourceLocation BeforeLoc) { + // Some statements (like AttributedStmt and ImplicitValueInitExpr) don't + // have valid source locations. Do not emit a gap region if this is the case + // in either AfterLoc end or BeforeLoc end. + if (AfterLoc.isInvalid() || BeforeLoc.isInvalid()) + return std::nullopt; + // If AfterLoc is in function-like macro, use the right parenthesis // location. if (AfterLoc.isMacroID()) { @@ -1321,7 +1406,7 @@ struct CounterCoverageMappingBuilder return; assert(SpellingRegion(SM, NewStartLoc, EndLoc).isInSourceOrder()); handleFileExit(NewStartLoc); - size_t Index = pushRegion({}, NewStartLoc, EndLoc); + size_t Index = pushRegion(Counter{}, NewStartLoc, EndLoc); getRegion().setSkipped(true); handleFileExit(EndLoc); popRegions(Index); @@ -1337,12 +1422,9 @@ struct CounterCoverageMappingBuilder CounterCoverageMappingBuilder( CoverageMappingModuleGen &CVM, llvm::DenseMap<const Stmt *, unsigned> &CounterMap, - llvm::DenseMap<const Stmt *, unsigned> &MCDCBitmapMap, - llvm::DenseMap<const Stmt *, MCDCConditionID> &CondIDMap, - SourceManager &SM, const LangOptions &LangOpts) + MCDC::State &MCDCState, SourceManager &SM, const LangOptions &LangOpts) : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap), - MCDCBitmapMap(MCDCBitmapMap), - MCDCBuilder(CVM.getCodeGenModule(), CondIDMap, MCDCBitmapMap) {} + MCDCState(MCDCState), MCDCBuilder(CVM.getCodeGenModule(), MCDCState) {} /// Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { @@ -1370,9 +1452,8 @@ struct CounterCoverageMappingBuilder for (const Stmt *Child : S->children()) if (Child) { // If last statement contains terminate statements, add a gap area - // between the two statements. Skipping attributed statements, because - // they don't have valid start location. - if (LastStmt && HasTerminateStmt && !isa<AttributedStmt>(Child)) { + // between the two statements. + if (LastStmt && HasTerminateStmt) { auto Gap = findGapAreaBetween(getEnd(LastStmt), getStart(Child)); if (Gap) fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), @@ -1439,6 +1520,10 @@ struct CounterCoverageMappingBuilder terminateRegion(S); } + void VisitCoroutineSuspendExpr(const CoroutineSuspendExpr *E) { + Visit(E->getOperand()); + } + void VisitCXXThrowExpr(const CXXThrowExpr *E) { extendRegion(E); if (E->getSubExpr()) @@ -1459,8 +1544,9 @@ struct CounterCoverageMappingBuilder void VisitBreakStmt(const BreakStmt *S) { assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); - BreakContinueStack.back().BreakCount = addCounters( - BreakContinueStack.back().BreakCount, getRegion().getCounter()); + if (!llvm::EnableSingleByteCoverage) + BreakContinueStack.back().BreakCount = addCounters( + BreakContinueStack.back().BreakCount, getRegion().getCounter()); // FIXME: a break in a switch should terminate regions for all preceding // case statements, not just the most recent one. terminateRegion(S); @@ -1468,8 +1554,9 @@ struct CounterCoverageMappingBuilder void VisitContinueStmt(const ContinueStmt *S) { assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); - BreakContinueStack.back().ContinueCount = addCounters( - BreakContinueStack.back().ContinueCount, getRegion().getCounter()); + if (!llvm::EnableSingleByteCoverage) + BreakContinueStack.back().ContinueCount = addCounters( + BreakContinueStack.back().ContinueCount, getRegion().getCounter()); terminateRegion(S); } @@ -1487,7 +1574,9 @@ struct CounterCoverageMappingBuilder extendRegion(S); Counter ParentCount = getRegion().getCounter(); - Counter BodyCount = getRegionCounter(S); + Counter BodyCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getBody()) + : getRegionCounter(S); // Handle the body first so that we can get the backedge count. BreakContinueStack.push_back(BreakContinue()); @@ -1500,7 +1589,9 @@ struct CounterCoverageMappingBuilder // Go back to handle the condition. Counter CondCount = - addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getCond()) + : addCounters(ParentCount, BackedgeCount, BC.ContinueCount); propagateCounts(CondCount, S->getCond()); adjustForOutOfOrderTraversal(getEnd(S)); @@ -1510,7 +1601,11 @@ struct CounterCoverageMappingBuilder fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), BodyCount); Counter OutCount = - addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + llvm::EnableSingleByteCoverage + ? getRegionCounter(S) + : addCounters(BC.BreakCount, + subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) { pushRegion(OutCount); GapRegionCounter = OutCount; @@ -1519,38 +1614,53 @@ struct CounterCoverageMappingBuilder } // Create Branch Region around condition. - createBranchRegion(S->getCond(), BodyCount, - subtractCounters(CondCount, BodyCount)); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(S->getCond(), BodyCount, + subtractCounters(CondCount, BodyCount)); } void VisitDoStmt(const DoStmt *S) { extendRegion(S); Counter ParentCount = getRegion().getCounter(); - Counter BodyCount = getRegionCounter(S); + Counter BodyCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getBody()) + : getRegionCounter(S); BreakContinueStack.push_back(BreakContinue()); extendRegion(S->getBody()); - Counter BackedgeCount = - propagateCounts(addCounters(ParentCount, BodyCount), S->getBody()); + + Counter BackedgeCount; + if (llvm::EnableSingleByteCoverage) + propagateCounts(BodyCount, S->getBody()); + else + BackedgeCount = + propagateCounts(addCounters(ParentCount, BodyCount), S->getBody()); + BreakContinue BC = BreakContinueStack.pop_back_val(); bool BodyHasTerminateStmt = HasTerminateStmt; HasTerminateStmt = false; - Counter CondCount = addCounters(BackedgeCount, BC.ContinueCount); + Counter CondCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getCond()) + : addCounters(BackedgeCount, BC.ContinueCount); propagateCounts(CondCount, S->getCond()); Counter OutCount = - addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + llvm::EnableSingleByteCoverage + ? getRegionCounter(S) + : addCounters(BC.BreakCount, + subtractCounters(CondCount, BodyCount)); if (OutCount != ParentCount) { pushRegion(OutCount); GapRegionCounter = OutCount; } // Create Branch Region around condition. - createBranchRegion(S->getCond(), BodyCount, - subtractCounters(CondCount, BodyCount)); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(S->getCond(), BodyCount, + subtractCounters(CondCount, BodyCount)); if (BodyHasTerminateStmt) HasTerminateStmt = true; @@ -1562,7 +1672,9 @@ struct CounterCoverageMappingBuilder Visit(S->getInit()); Counter ParentCount = getRegion().getCounter(); - Counter BodyCount = getRegionCounter(S); + Counter BodyCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getBody()) + : getRegionCounter(S); // The loop increment may contain a break or continue. if (S->getInc()) @@ -1581,14 +1693,23 @@ struct CounterCoverageMappingBuilder // the count for all the continue statements. BreakContinue IncrementBC; if (const Stmt *Inc = S->getInc()) { - propagateCounts(addCounters(BackedgeCount, BodyBC.ContinueCount), Inc); + Counter IncCount; + if (llvm::EnableSingleByteCoverage) + IncCount = getRegionCounter(S->getInc()); + else + IncCount = addCounters(BackedgeCount, BodyBC.ContinueCount); + propagateCounts(IncCount, Inc); IncrementBC = BreakContinueStack.pop_back_val(); } // Go back to handle the condition. - Counter CondCount = addCounters( - addCounters(ParentCount, BackedgeCount, BodyBC.ContinueCount), - IncrementBC.ContinueCount); + Counter CondCount = + llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getCond()) + : addCounters( + addCounters(ParentCount, BackedgeCount, BodyBC.ContinueCount), + IncrementBC.ContinueCount); + if (const Expr *Cond = S->getCond()) { propagateCounts(CondCount, Cond); adjustForOutOfOrderTraversal(getEnd(S)); @@ -1599,8 +1720,11 @@ struct CounterCoverageMappingBuilder if (Gap) fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), BodyCount); - Counter OutCount = addCounters(BodyBC.BreakCount, IncrementBC.BreakCount, - subtractCounters(CondCount, BodyCount)); + Counter OutCount = + llvm::EnableSingleByteCoverage + ? getRegionCounter(S) + : addCounters(BodyBC.BreakCount, IncrementBC.BreakCount, + subtractCounters(CondCount, BodyCount)); if (OutCount != ParentCount) { pushRegion(OutCount); GapRegionCounter = OutCount; @@ -1609,8 +1733,9 @@ struct CounterCoverageMappingBuilder } // Create Branch Region around condition. - createBranchRegion(S->getCond(), BodyCount, - subtractCounters(CondCount, BodyCount)); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(S->getCond(), BodyCount, + subtractCounters(CondCount, BodyCount)); } void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { @@ -1621,7 +1746,9 @@ struct CounterCoverageMappingBuilder Visit(S->getRangeStmt()); Counter ParentCount = getRegion().getCounter(); - Counter BodyCount = getRegionCounter(S); + Counter BodyCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getBody()) + : getRegionCounter(S); BreakContinueStack.push_back(BreakContinue()); extendRegion(S->getBody()); @@ -1636,10 +1763,15 @@ struct CounterCoverageMappingBuilder if (Gap) fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), BodyCount); - Counter LoopCount = - addCounters(ParentCount, BackedgeCount, BC.ContinueCount); - Counter OutCount = - addCounters(BC.BreakCount, subtractCounters(LoopCount, BodyCount)); + Counter OutCount; + Counter LoopCount; + if (llvm::EnableSingleByteCoverage) + OutCount = getRegionCounter(S); + else { + LoopCount = addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + OutCount = + addCounters(BC.BreakCount, subtractCounters(LoopCount, BodyCount)); + } if (OutCount != ParentCount) { pushRegion(OutCount); GapRegionCounter = OutCount; @@ -1648,8 +1780,9 @@ struct CounterCoverageMappingBuilder } // Create Branch Region around condition. - createBranchRegion(S->getCond(), BodyCount, - subtractCounters(LoopCount, BodyCount)); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(S->getCond(), BodyCount, + subtractCounters(LoopCount, BodyCount)); } void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { @@ -1710,7 +1843,7 @@ struct CounterCoverageMappingBuilder propagateCounts(Counter::getZero(), Body); BreakContinue BC = BreakContinueStack.pop_back_val(); - if (!BreakContinueStack.empty()) + if (!BreakContinueStack.empty() && !llvm::EnableSingleByteCoverage) BreakContinueStack.back().ContinueCount = addCounters( BreakContinueStack.back().ContinueCount, BC.ContinueCount); @@ -1725,6 +1858,11 @@ struct CounterCoverageMappingBuilder MostRecentLocation = getStart(S); handleFileExit(ExitLoc); + // When single byte coverage mode is enabled, do not create branch region by + // early returning. + if (llvm::EnableSingleByteCoverage) + return; + // Create a Branch Region around each Case. Subtract the case's // counter from the Parent counter to track the "False" branch count. Counter CaseCountSum; @@ -1757,8 +1895,10 @@ struct CounterCoverageMappingBuilder extendRegion(S); SourceMappingRegion &Parent = getRegion(); + Counter Count = llvm::EnableSingleByteCoverage + ? getRegionCounter(S) + : addCounters(Parent.getCounter(), getRegionCounter(S)); - Counter Count = addCounters(Parent.getCounter(), getRegionCounter(S)); // Reuse the existing region if it starts at our label. This is typical of // the first case in a switch. if (Parent.hasStartLoc() && Parent.getBeginLoc() == getStart(S)) @@ -1876,7 +2016,9 @@ struct CounterCoverageMappingBuilder extendRegion(S->getCond()); Counter ParentCount = getRegion().getCounter(); - Counter ThenCount = getRegionCounter(S); + Counter ThenCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(S->getThen()) + : getRegionCounter(S); // Emitting a counter for the condition makes it easier to interpret the // counter for the body when looking at the coverage. @@ -1890,7 +2032,12 @@ struct CounterCoverageMappingBuilder extendRegion(S->getThen()); Counter OutCount = propagateCounts(ThenCount, S->getThen()); - Counter ElseCount = subtractCounters(ParentCount, ThenCount); + + Counter ElseCount; + if (!llvm::EnableSingleByteCoverage) + ElseCount = subtractCounters(ParentCount, ThenCount); + else if (S->getElse()) + ElseCount = getRegionCounter(S->getElse()); if (const Stmt *Else = S->getElse()) { bool ThenHasTerminateStmt = HasTerminateStmt; @@ -1901,21 +2048,28 @@ struct CounterCoverageMappingBuilder if (Gap) fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), ElseCount); extendRegion(Else); - OutCount = addCounters(OutCount, propagateCounts(ElseCount, Else)); + + Counter ElseOutCount = propagateCounts(ElseCount, Else); + if (!llvm::EnableSingleByteCoverage) + OutCount = addCounters(OutCount, ElseOutCount); if (ThenHasTerminateStmt) HasTerminateStmt = true; - } else + } else if (!llvm::EnableSingleByteCoverage) OutCount = addCounters(OutCount, ElseCount); + if (llvm::EnableSingleByteCoverage) + OutCount = getRegionCounter(S); + if (OutCount != ParentCount) { pushRegion(OutCount); GapRegionCounter = OutCount; } - // Create Branch Region around condition. - createBranchRegion(S->getCond(), ThenCount, - subtractCounters(ParentCount, ThenCount)); + if (!S->isConsteval() && !llvm::EnableSingleByteCoverage) + // Create Branch Region around condition. + createBranchRegion(S->getCond(), ThenCount, + subtractCounters(ParentCount, ThenCount)); } void VisitCXXTryStmt(const CXXTryStmt *S) { @@ -1941,12 +2095,16 @@ struct CounterCoverageMappingBuilder extendRegion(E); Counter ParentCount = getRegion().getCounter(); - Counter TrueCount = getRegionCounter(E); - - propagateCounts(ParentCount, E->getCond()); + Counter TrueCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(E->getTrueExpr()) + : getRegionCounter(E); Counter OutCount; - if (!isa<BinaryConditionalOperator>(E)) { + if (const auto *BCO = dyn_cast<BinaryConditionalOperator>(E)) { + propagateCounts(ParentCount, BCO->getCommon()); + OutCount = TrueCount; + } else { + propagateCounts(ParentCount, E->getCond()); // The 'then' count applies to the area immediately after the condition. auto Gap = findGapAreaBetween(E->getQuestionLoc(), getStart(E->getTrueExpr())); @@ -1958,9 +2116,15 @@ struct CounterCoverageMappingBuilder } extendRegion(E->getFalseExpr()); - OutCount = addCounters( - OutCount, propagateCounts(subtractCounters(ParentCount, TrueCount), - E->getFalseExpr())); + Counter FalseCount = llvm::EnableSingleByteCoverage + ? getRegionCounter(E->getFalseExpr()) + : subtractCounters(ParentCount, TrueCount); + + Counter FalseOutCount = propagateCounts(FalseCount, E->getFalseExpr()); + if (llvm::EnableSingleByteCoverage) + OutCount = getRegionCounter(E); + else + OutCount = addCounters(OutCount, FalseOutCount); if (OutCount != ParentCount) { pushRegion(OutCount); @@ -1968,13 +2132,93 @@ struct CounterCoverageMappingBuilder } // Create Branch Region around condition. - createBranchRegion(E->getCond(), TrueCount, - subtractCounters(ParentCount, TrueCount)); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(E->getCond(), TrueCount, + subtractCounters(ParentCount, TrueCount)); + } + + void createOrCancelDecision(const BinaryOperator *E, unsigned Since) { + unsigned NumConds = MCDCBuilder.getTotalConditionsAndReset(E); + if (NumConds == 0) + return; + + // Extract [ID, Conds] to construct the graph. + llvm::SmallVector<mcdc::ConditionIDs> CondIDs(NumConds); + for (const auto &SR : ArrayRef(SourceRegions).slice(Since)) { + if (SR.isMCDCBranch()) { + auto [ID, Conds] = SR.getMCDCBranchParams(); + CondIDs[ID] = Conds; + } + } + + // Construct the graph and calculate `Indices`. + mcdc::TVIdxBuilder Builder(CondIDs); + unsigned NumTVs = Builder.NumTestVectors; + unsigned MaxTVs = CVM.getCodeGenModule().getCodeGenOpts().MCDCMaxTVs; + assert(MaxTVs < mcdc::TVIdxBuilder::HardMaxTVs); + + if (NumTVs > MaxTVs) { + // NumTVs exceeds MaxTVs -- warn and cancel the Decision. + cancelDecision(E, Since, NumTVs, MaxTVs); + return; + } + + // Update the state for CodeGenPGO + assert(MCDCState.DecisionByStmt.contains(E)); + MCDCState.DecisionByStmt[E] = { + MCDCState.BitmapBits, // Top + std::move(Builder.Indices), + }; + + auto DecisionParams = mcdc::DecisionParameters{ + MCDCState.BitmapBits += NumTVs, // Tail + NumConds, + }; + + // Create MCDC Decision Region. + createDecisionRegion(E, DecisionParams); + } + + // Warn and cancel the Decision. + void cancelDecision(const BinaryOperator *E, unsigned Since, int NumTVs, + int MaxTVs) { + auto &Diag = CVM.getCodeGenModule().getDiags(); + unsigned DiagID = + Diag.getCustomDiagID(DiagnosticsEngine::Warning, + "unsupported MC/DC boolean expression; " + "number of test vectors (%0) exceeds max (%1). " + "Expression will not be covered"); + Diag.Report(E->getBeginLoc(), DiagID) << NumTVs << MaxTVs; + + // Restore MCDCBranch to Branch. + for (auto &SR : MutableArrayRef(SourceRegions).slice(Since)) { + assert(!SR.isMCDCDecision() && "Decision shouldn't be seen here"); + if (SR.isMCDCBranch()) + SR.resetMCDCParams(); + } + + // Tell CodeGenPGO not to instrument. + MCDCState.DecisionByStmt.erase(E); + } + + /// Check if E belongs to system headers. + bool isExprInSystemHeader(const BinaryOperator *E) const { + return (!SystemHeadersCoverage && + SM.isInSystemHeader(SM.getSpellingLoc(E->getOperatorLoc())) && + SM.isInSystemHeader(SM.getSpellingLoc(E->getBeginLoc())) && + SM.isInSystemHeader(SM.getSpellingLoc(E->getEndLoc()))); } void VisitBinLAnd(const BinaryOperator *E) { + if (isExprInSystemHeader(E)) { + LeafExprSet.insert(E); + return; + } + bool IsRootNode = MCDCBuilder.isIdle(); + unsigned SourceRegionsSince = SourceRegions.size(); + // Keep track of Binary Operator and assign MCDC condition IDs. MCDCBuilder.pushAndAssignIDs(E); @@ -1992,11 +2236,6 @@ struct CounterCoverageMappingBuilder // Track RHS True/False Decision. const auto DecisionRHS = MCDCBuilder.back(); - // Create MCDC Decision Region if at top-level (root). - unsigned NumConds = 0; - if (IsRootNode && (NumConds = MCDCBuilder.getTotalConditionsAndReset(E))) - createDecisionRegion(E, getRegionBitmap(E), NumConds); - // Extract the RHS's Execution Counter. Counter RHSExecCnt = getRegionCounter(E); @@ -2007,12 +2246,18 @@ struct CounterCoverageMappingBuilder Counter ParentCnt = getRegion().getCounter(); // Create Branch Region around LHS condition. - createBranchRegion(E->getLHS(), RHSExecCnt, - subtractCounters(ParentCnt, RHSExecCnt), DecisionLHS); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(E->getLHS(), RHSExecCnt, + subtractCounters(ParentCnt, RHSExecCnt), DecisionLHS); // Create Branch Region around RHS condition. - createBranchRegion(E->getRHS(), RHSTrueCnt, - subtractCounters(RHSExecCnt, RHSTrueCnt), DecisionRHS); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(E->getRHS(), RHSTrueCnt, + subtractCounters(RHSExecCnt, RHSTrueCnt), DecisionRHS); + + // Create MCDC Decision Region if at top-level (root). + if (IsRootNode) + createOrCancelDecision(E, SourceRegionsSince); } // Determine whether the right side of OR operation need to be visited. @@ -2026,8 +2271,15 @@ struct CounterCoverageMappingBuilder } void VisitBinLOr(const BinaryOperator *E) { + if (isExprInSystemHeader(E)) { + LeafExprSet.insert(E); + return; + } + bool IsRootNode = MCDCBuilder.isIdle(); + unsigned SourceRegionsSince = SourceRegions.size(); + // Keep track of Binary Operator and assign MCDC condition IDs. MCDCBuilder.pushAndAssignIDs(E); @@ -2045,11 +2297,6 @@ struct CounterCoverageMappingBuilder // Track RHS True/False Decision. const auto DecisionRHS = MCDCBuilder.back(); - // Create MCDC Decision Region if at top-level (root). - unsigned NumConds = 0; - if (IsRootNode && (NumConds = MCDCBuilder.getTotalConditionsAndReset(E))) - createDecisionRegion(E, getRegionBitmap(E), NumConds); - // Extract the RHS's Execution Counter. Counter RHSExecCnt = getRegionCounter(E); @@ -2064,12 +2311,18 @@ struct CounterCoverageMappingBuilder Counter ParentCnt = getRegion().getCounter(); // Create Branch Region around LHS condition. - createBranchRegion(E->getLHS(), subtractCounters(ParentCnt, RHSExecCnt), - RHSExecCnt, DecisionLHS); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(E->getLHS(), subtractCounters(ParentCnt, RHSExecCnt), + RHSExecCnt, DecisionLHS); // Create Branch Region around RHS condition. - createBranchRegion(E->getRHS(), subtractCounters(RHSExecCnt, RHSFalseCnt), - RHSFalseCnt, DecisionRHS); + if (!llvm::EnableSingleByteCoverage) + createBranchRegion(E->getRHS(), subtractCounters(RHSExecCnt, RHSFalseCnt), + RHSFalseCnt, DecisionRHS); + + // Create MCDC Decision Region if at top-level (root). + if (IsRootNode) + createOrCancelDecision(E, SourceRegionsSince); } void VisitLambdaExpr(const LambdaExpr *LE) { @@ -2077,13 +2330,18 @@ struct CounterCoverageMappingBuilder // propagate counts into them. } + void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *AILE) { + Visit(AILE->getCommonExpr()->getSourceExpr()); + } + void VisitPseudoObjectExpr(const PseudoObjectExpr *POE) { // Just visit syntatic expression as this is what users actually write. VisitStmt(POE->getSyntacticForm()); } void VisitOpaqueValueExpr(const OpaqueValueExpr* OVE) { - Visit(OVE->getSourceExpr()); + if (OVE->isUnique()) + Visit(OVE->getSourceExpr()); } }; @@ -2120,9 +2378,10 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName, OS << "File " << R.FileID << ", " << R.LineStart << ":" << R.ColumnStart << " -> " << R.LineEnd << ":" << R.ColumnEnd << " = "; - if (R.Kind == CounterMappingRegion::MCDCDecisionRegion) { - OS << "M:" << R.MCDCParams.BitmapIdx; - OS << ", C:" << R.MCDCParams.NumConditions; + if (const auto *DecisionParams = + std::get_if<mcdc::DecisionParameters>(&R.MCDCParams)) { + OS << "M:" << DecisionParams->BitmapIdx; + OS << ", C:" << DecisionParams->NumConditions; } else { Ctx.dump(R.Count, OS); @@ -2133,9 +2392,11 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName, } } - if (R.Kind == CounterMappingRegion::MCDCBranchRegion) { - OS << " [" << R.MCDCParams.ID << "," << R.MCDCParams.TrueID; - OS << "," << R.MCDCParams.FalseID << "] "; + if (const auto *BranchParams = + std::get_if<mcdc::BranchParameters>(&R.MCDCParams)) { + OS << " [" << BranchParams->ID + 1 << "," + << BranchParams->Conds[true] + 1; + OS << "," << BranchParams->Conds[false] + 1 << "] "; } if (R.Kind == CounterMappingRegion::ExpansionRegion) @@ -2344,9 +2605,9 @@ unsigned CoverageMappingModuleGen::getFileID(FileEntryRef File) { void CoverageMappingGen::emitCounterMapping(const Decl *D, llvm::raw_ostream &OS) { - assert(CounterMap && MCDCBitmapMap); - CounterCoverageMappingBuilder Walker(CVM, *CounterMap, *MCDCBitmapMap, - *CondIDMap, SM, LangOpts); + assert(CounterMap && MCDCState); + CounterCoverageMappingBuilder Walker(CVM, *CounterMap, *MCDCState, SM, + LangOpts); Walker.VisitDecl(D); Walker.write(OS); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.h b/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.h index 62cea173c9fc..fe4b93f3af85 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.h +++ b/contrib/llvm-project/clang/lib/CodeGen/CoverageMappingGen.h @@ -19,8 +19,13 @@ #include "clang/Lex/Preprocessor.h" #include "llvm/ADT/DenseMap.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" +namespace llvm::coverage { +extern cl::opt<bool> SystemHeadersCoverage; +} + namespace clang { class LangOptions; @@ -91,6 +96,10 @@ namespace CodeGen { class CodeGenModule; +namespace MCDC { +struct State; +} + /// Organizes the cross-function state that is used while generating /// code coverage mapping data. class CoverageMappingModuleGen { @@ -150,22 +159,20 @@ class CoverageMappingGen { SourceManager &SM; const LangOptions &LangOpts; llvm::DenseMap<const Stmt *, unsigned> *CounterMap; - llvm::DenseMap<const Stmt *, unsigned> *MCDCBitmapMap; - llvm::DenseMap<const Stmt *, unsigned> *CondIDMap; + MCDC::State *MCDCState; public: CoverageMappingGen(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts) : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(nullptr), - MCDCBitmapMap(nullptr), CondIDMap(nullptr) {} + MCDCState(nullptr) {} CoverageMappingGen(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts, llvm::DenseMap<const Stmt *, unsigned> *CounterMap, - llvm::DenseMap<const Stmt *, unsigned> *MCDCBitmapMap, - llvm::DenseMap<const Stmt *, unsigned> *CondIDMap) + MCDC::State *MCDCState) : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(CounterMap), - MCDCBitmapMap(MCDCBitmapMap), CondIDMap(CondIDMap) {} + MCDCState(MCDCState) {} /// Emit the coverage mapping data which maps the regions of /// code to counters that will be used to find the execution diff --git a/contrib/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp index d173806ec8ce..0be92fb2e275 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -23,6 +23,7 @@ #include "CGVTables.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantEmitter.h" #include "TargetInfo.h" #include "clang/AST/Attr.h" #include "clang/AST/Mangle.h" @@ -178,7 +179,7 @@ public: return CatchTypeInfo{getAddrOfRTTIDescriptor(Ty), 0}; } - bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; + bool shouldTypeidBeNullChecked(QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, Address ThisPtr, @@ -307,10 +308,6 @@ public: CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase); - llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) override; - llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -340,9 +337,11 @@ public: bool exportThunk() override { return true; } llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, - const ThisAdjustment &TA) override; + const CXXRecordDecl *UnadjustedThisClass, + const ThunkInfo &TI) override; llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, + const CXXRecordDecl *UnadjustedRetClass, const ReturnAdjustment &RA) override; size_t getSrcArgforCopyCtor(const CXXConstructorDecl *, @@ -389,6 +388,9 @@ public: bool NeedsVTTParameter(GlobalDecl GD) override; + llvm::Constant * + getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD); + /**************************** RTTI Uniqueness ******************************/ protected: @@ -427,6 +429,9 @@ public: const CXXRecordDecl *RD) override; private: + llvm::Constant * + getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD); + bool hasAnyUnusedVirtualInlineFunction(const CXXRecordDecl *RD) const { const auto &VtableLayout = CGM.getItaniumVTableContext().getVTableLayout(RD); @@ -646,7 +651,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Apply the adjustment and cast back to the original struct type // for consistency. - llvm::Value *This = ThisAddr.getPointer(); + llvm::Value *This = ThisAddr.emitRawPointer(CGF); This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj); ThisPtrForCall = This; @@ -836,7 +841,25 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( CalleePtr->addIncoming(VirtualFn, FnVirtual); CalleePtr->addIncoming(NonVirtualFn, FnNonVirtual); - CGCallee Callee(FPT, CalleePtr); + CGPointerAuthInfo PointerAuth; + + if (const auto &Schema = + CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers) { + llvm::PHINode *DiscriminatorPHI = Builder.CreatePHI(CGF.IntPtrTy, 2); + DiscriminatorPHI->addIncoming(llvm::ConstantInt::get(CGF.IntPtrTy, 0), + FnVirtual); + const auto &AuthInfo = + CGM.getMemberFunctionPointerAuthInfo(QualType(MPT, 0)); + assert(Schema.getKey() == AuthInfo.getKey() && + "Keys for virtual and non-virtual member functions must match"); + auto *NonVirtualDiscriminator = AuthInfo.getDiscriminator(); + DiscriminatorPHI->addIncoming(NonVirtualDiscriminator, FnNonVirtual); + PointerAuth = CGPointerAuthInfo( + Schema.getKey(), Schema.getAuthenticationMode(), Schema.isIsaPointer(), + Schema.authenticatesNullValues(), DiscriminatorPHI); + } + + CGCallee Callee(FPT, CalleePtr, PointerAuth); return Callee; } @@ -850,10 +873,29 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress( CGBuilderTy &Builder = CGF.Builder; // Apply the offset, which we assume is non-null. - return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.getPointer(), MemPtr, + return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr, "memptr.offset"); } +// See if it's possible to return a constant signed pointer. +static llvm::Constant *pointerAuthResignConstant( + llvm::Value *Ptr, const CGPointerAuthInfo &CurAuthInfo, + const CGPointerAuthInfo &NewAuthInfo, CodeGenModule &CGM) { + const auto *CPA = dyn_cast<llvm::ConstantPtrAuth>(Ptr); + + if (!CPA) + return nullptr; + + assert(CPA->getKey()->getZExtValue() == CurAuthInfo.getKey() && + CPA->getAddrDiscriminator()->isZeroValue() && + CPA->getDiscriminator() == CurAuthInfo.getDiscriminator() && + "unexpected key or discriminators"); + + return CGM.getConstantSignedPointer( + CPA->getPointer(), NewAuthInfo.getKey(), nullptr, + cast<llvm::ConstantInt>(NewAuthInfo.getDiscriminator())); +} + /// Perform a bitcast, derived-to-base, or base-to-derived member pointer /// conversion. /// @@ -881,21 +923,63 @@ llvm::Value * ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF, const CastExpr *E, llvm::Value *src) { + // Use constant emission if we can. + if (isa<llvm::Constant>(src)) + return EmitMemberPointerConversion(E, cast<llvm::Constant>(src)); + assert(E->getCastKind() == CK_DerivedToBaseMemberPointer || E->getCastKind() == CK_BaseToDerivedMemberPointer || E->getCastKind() == CK_ReinterpretMemberPointer); + CGBuilderTy &Builder = CGF.Builder; + QualType DstType = E->getType(); + + if (DstType->isMemberFunctionPointerType()) { + if (const auto &NewAuthInfo = + CGM.getMemberFunctionPointerAuthInfo(DstType)) { + QualType SrcType = E->getSubExpr()->getType(); + assert(SrcType->isMemberFunctionPointerType()); + const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(SrcType); + llvm::Value *MemFnPtr = Builder.CreateExtractValue(src, 0, "memptr.ptr"); + llvm::Type *OrigTy = MemFnPtr->getType(); + + llvm::BasicBlock *StartBB = Builder.GetInsertBlock(); + llvm::BasicBlock *ResignBB = CGF.createBasicBlock("resign"); + llvm::BasicBlock *MergeBB = CGF.createBasicBlock("merge"); + + // Check whether we have a virtual offset or a pointer to a function. + assert(UseARMMethodPtrABI && "ARM ABI expected"); + llvm::Value *Adj = Builder.CreateExtractValue(src, 1, "memptr.adj"); + llvm::Constant *Ptrdiff_1 = llvm::ConstantInt::get(CGM.PtrDiffTy, 1); + llvm::Value *AndVal = Builder.CreateAnd(Adj, Ptrdiff_1); + llvm::Value *IsVirtualOffset = + Builder.CreateIsNotNull(AndVal, "is.virtual.offset"); + Builder.CreateCondBr(IsVirtualOffset, MergeBB, ResignBB); + + CGF.EmitBlock(ResignBB); + llvm::Type *PtrTy = llvm::PointerType::getUnqual(CGM.Int8Ty); + MemFnPtr = Builder.CreateIntToPtr(MemFnPtr, PtrTy); + MemFnPtr = + CGF.emitPointerAuthResign(MemFnPtr, SrcType, CurAuthInfo, NewAuthInfo, + isa<llvm::Constant>(src)); + MemFnPtr = Builder.CreatePtrToInt(MemFnPtr, OrigTy); + llvm::Value *ResignedVal = Builder.CreateInsertValue(src, MemFnPtr, 0); + ResignBB = Builder.GetInsertBlock(); + + CGF.EmitBlock(MergeBB); + llvm::PHINode *NewSrc = Builder.CreatePHI(src->getType(), 2); + NewSrc->addIncoming(src, StartBB); + NewSrc->addIncoming(ResignedVal, ResignBB); + src = NewSrc; + } + } + // Under Itanium, reinterprets don't require any additional processing. if (E->getCastKind() == CK_ReinterpretMemberPointer) return src; - // Use constant emission if we can. - if (isa<llvm::Constant>(src)) - return EmitMemberPointerConversion(E, cast<llvm::Constant>(src)); - llvm::Constant *adj = getMemberPointerAdjustment(E); if (!adj) return src; - CGBuilderTy &Builder = CGF.Builder; bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer); const MemberPointerType *destTy = @@ -933,6 +1017,34 @@ ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF, return Builder.CreateInsertValue(src, dstAdj, 1); } +static llvm::Constant * +pointerAuthResignMemberFunctionPointer(llvm::Constant *Src, QualType DestType, + QualType SrcType, CodeGenModule &CGM) { + assert(DestType->isMemberFunctionPointerType() && + SrcType->isMemberFunctionPointerType() && + "member function pointers expected"); + if (DestType == SrcType) + return Src; + + const auto &NewAuthInfo = CGM.getMemberFunctionPointerAuthInfo(DestType); + const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(SrcType); + + if (!NewAuthInfo && !CurAuthInfo) + return Src; + + llvm::Constant *MemFnPtr = Src->getAggregateElement(0u); + if (MemFnPtr->getNumOperands() == 0) { + // src must be a pair of null pointers. + assert(isa<llvm::ConstantInt>(MemFnPtr) && "constant int expected"); + return Src; + } + + llvm::Constant *ConstPtr = pointerAuthResignConstant( + cast<llvm::User>(MemFnPtr)->getOperand(0), CurAuthInfo, NewAuthInfo, CGM); + ConstPtr = llvm::ConstantExpr::getPtrToInt(ConstPtr, MemFnPtr->getType()); + return ConstantFoldInsertValueInstruction(Src, ConstPtr, 0); +} + llvm::Constant * ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E, llvm::Constant *src) { @@ -940,6 +1052,12 @@ ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E, E->getCastKind() == CK_BaseToDerivedMemberPointer || E->getCastKind() == CK_ReinterpretMemberPointer); + QualType DstType = E->getType(); + + if (DstType->isMemberFunctionPointerType()) + src = pointerAuthResignMemberFunctionPointer( + src, DstType, E->getSubExpr()->getType(), CGM); + // Under Itanium, reinterprets don't require any additional processing. if (E->getCastKind() == CK_ReinterpretMemberPointer) return src; @@ -1037,9 +1155,32 @@ llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD, // least significant bit of adj then makes exactly the same // discrimination as the least significant bit of ptr does for // Itanium. - MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset); - MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy, - 2 * ThisAdjustment.getQuantity() + 1); + + // We cannot use the Itanium ABI's representation for virtual member + // function pointers under pointer authentication because it would + // require us to store both the virtual offset and the constant + // discriminator in the pointer, which would be immediately vulnerable + // to attack. Instead we introduce a thunk that does the virtual dispatch + // and store it as if it were a non-virtual member function. This means + // that virtual function pointers may not compare equal anymore, but + // fortunately they aren't required to by the standard, and we do make + // a best-effort attempt to re-use the thunk. + // + // To support interoperation with code in which pointer authentication + // is disabled, derefencing a member function pointer must still handle + // the virtual case, but it can use a discriminator which should never + // be valid. + const auto &Schema = + CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers; + if (Schema) + MemPtr[0] = llvm::ConstantExpr::getPtrToInt( + getSignedVirtualMemberFunctionPointer(MD), CGM.PtrDiffTy); + else + MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset); + // Don't set the LSB of adj to 1 if pointer authentication for member + // function pointers is enabled. + MemPtr[1] = llvm::ConstantInt::get( + CGM.PtrDiffTy, 2 * ThisAdjustment.getQuantity() + !Schema); } else { // Itanium C++ ABI 2.3: // For a virtual function, [the pointer field] is 1 plus the @@ -1061,7 +1202,7 @@ llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD, // function type is incomplete. Ty = CGM.PtrDiffTy; } - llvm::Constant *addr = CGM.GetAddrOfFunction(MD, Ty); + llvm::Constant *addr = CGM.getMemberFunctionPointer(MD, Ty); MemPtr[0] = llvm::ConstantExpr::getPtrToInt(addr, CGM.PtrDiffTy); MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy, @@ -1081,8 +1222,12 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP, CharUnits ThisAdjustment = getContext().getMemberPointerPathAdjustment(MP); - if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) - return BuildMemberPointer(MD, ThisAdjustment); + if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) { + llvm::Constant *Src = BuildMemberPointer(MD, ThisAdjustment); + QualType SrcType = getContext().getMemberPointerType( + MD->getType(), MD->getParent()->getTypeForDecl()); + return pointerAuthResignMemberFunctionPointer(Src, MPType, SrcType, CGM); + } CharUnits FieldOffset = getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD)); @@ -1245,7 +1390,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, CGF.getPointerAlign()); // Apply the offset. - llvm::Value *CompletePtr = Ptr.getPointer(); + llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF); CompletePtr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset); @@ -1322,8 +1467,16 @@ void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) { CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl()); if (!Record->hasTrivialDestructor()) { + // __cxa_throw is declared to take its destructor as void (*)(void *). We + // must match that if function pointers can be authenticated with a + // discriminator based on their type. + const ASTContext &Ctx = getContext(); + QualType DtorTy = Ctx.getFunctionType(Ctx.VoidTy, {Ctx.VoidPtrTy}, + FunctionProtoType::ExtProtoInfo()); + CXXDestructorDecl *DtorD = Record->getDestructor(); Dtor = CGM.getAddrOfCXXStructor(GlobalDecl(DtorD, Dtor_Complete)); + Dtor = CGM.getFunctionPointer(Dtor, DtorTy); } } if (!Dtor) Dtor = llvm::Constant::getNullValue(CGM.Int8PtrTy); @@ -1347,9 +1500,10 @@ static llvm::FunctionCallee getItaniumDynamicCastFn(CodeGenFunction &CGF) { llvm::FunctionType *FTy = llvm::FunctionType::get(Int8PtrTy, Args, false); - // Mark the function as nounwind readonly. + // Mark the function as nounwind willreturn readonly. llvm::AttrBuilder FuncAttrs(CGF.getLLVMContext()); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); + FuncAttrs.addAttribute(llvm::Attribute::WillReturn); FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly()); llvm::AttributeList Attrs = llvm::AttributeList::get( CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, FuncAttrs); @@ -1422,9 +1576,8 @@ static llvm::FunctionCallee getBadTypeidFn(CodeGenFunction &CGF) { return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_typeid"); } -bool ItaniumCXXABI::shouldTypeidBeNullChecked(bool IsDeref, - QualType SrcRecordTy) { - return IsDeref; +bool ItaniumCXXABI::shouldTypeidBeNullChecked(QualType SrcRecordTy) { + return true; } void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) { @@ -1481,9 +1634,22 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall( computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity()); // Emit the call to __dynamic_cast. - llvm::Value *Args[] = {ThisAddr.getPointer(), SrcRTTI, DestRTTI, OffsetHint}; - llvm::Value *Value = - CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), Args); + llvm::Value *Value = ThisAddr.emitRawPointer(CGF); + if (CGM.getCodeGenOpts().PointerAuth.CXXVTablePointers) { + // We perform a no-op load of the vtable pointer here to force an + // authentication. In environments that do not support pointer + // authentication this is a an actual no-op that will be elided. When + // pointer authentication is supported and enforced on vtable pointers this + // load can trap. + llvm::Value *Vtable = + CGF.GetVTablePtr(ThisAddr, CGM.Int8PtrTy, SrcDecl, + CodeGenFunction::VTableAuthMode::MustTrap); + assert(Vtable); + (void)Vtable; + } + + llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint}; + Value = CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), args); /// C++ [expr.dynamic.cast]p9: /// A failed cast to reference type throws std::bad_cast @@ -1570,7 +1736,7 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast( VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy)); llvm::Value *Success = CGF.Builder.CreateICmpEQ( VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl)); - llvm::Value *Result = ThisAddr.getPointer(); + llvm::Value *Result = ThisAddr.emitRawPointer(CGF); if (!Offset->isZero()) Result = CGF.Builder.CreateInBoundsGEP( CGF.CharTy, Result, @@ -1610,7 +1776,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top"); } // Finally, add the offset to the pointer. - return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.getPointer(), + return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF), OffsetToTop); } @@ -1791,8 +1957,39 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, else Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD); - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, VTT, VTTTy, - nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), + ThisTy, VTT, VTTTy, nullptr); +} + +// Check if any non-inline method has the specified attribute. +template <typename T> +static bool CXXRecordNonInlineHasAttr(const CXXRecordDecl *RD) { + for (const auto *D : RD->noload_decls()) { + if (const auto *FD = dyn_cast<FunctionDecl>(D)) { + if (FD->isInlined() || FD->doesThisDeclarationHaveABody() || + FD->isPureVirtual()) + continue; + if (D->hasAttr<T>()) + return true; + } + } + + return false; +} + +static void setVTableSelectiveDLLImportExport(CodeGenModule &CGM, + llvm::GlobalVariable *VTable, + const CXXRecordDecl *RD) { + if (VTable->getDLLStorageClass() != + llvm::GlobalVariable::DefaultStorageClass || + RD->hasAttr<DLLImportAttr>() || RD->hasAttr<DLLExportAttr>()) + return; + + if (CGM.getVTables().isVTableExternal(RD)) { + if (CXXRecordNonInlineHasAttr<DLLImportAttr>(RD)) + VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + } else if (CXXRecordNonInlineHasAttr<DLLExportAttr>(RD)) + VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); } void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1820,6 +2017,9 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, if (CGM.supportsCOMDAT() && VTable->isWeakForLinker()) VTable->setComdat(CGM.getModule().getOrInsertComdat(VTable->getName())); + if (CGM.getTarget().hasPS4DLLImportExport()) + setVTableSelectiveDLLImportExport(CGM, VTable, RD); + // Set the right visibility. CGM.setGVProperties(VTable, RD); @@ -1884,42 +2084,27 @@ ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base, // Find the appropriate vtable within the vtable group, and the address point // within that vtable. + const VTableLayout &Layout = + CGM.getItaniumVTableContext().getVTableLayout(VTableClass); VTableLayout::AddressPointLocation AddressPoint = - CGM.getItaniumVTableContext() - .getVTableLayout(VTableClass) - .getAddressPoint(Base); + Layout.getAddressPoint(Base); llvm::Value *Indices[] = { llvm::ConstantInt::get(CGM.Int32Ty, 0), llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.VTableIndex), llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.AddressPointIndex), }; - return llvm::ConstantExpr::getGetElementPtr(VTable->getValueType(), VTable, - Indices, /*InBounds=*/true, - /*InRangeIndex=*/1); -} - -// Check whether all the non-inline virtual methods for the class have the -// specified attribute. -template <typename T> -static bool CXXRecordAllNonInlineVirtualsHaveAttr(const CXXRecordDecl *RD) { - bool FoundNonInlineVirtualMethodWithAttr = false; - for (const auto *D : RD->noload_decls()) { - if (const auto *FD = dyn_cast<FunctionDecl>(D)) { - if (!FD->isVirtualAsWritten() || FD->isInlineSpecified() || - FD->doesThisDeclarationHaveABody()) - continue; - if (!D->hasAttr<T>()) - return false; - FoundNonInlineVirtualMethodWithAttr = true; - } - } - - // We didn't find any non-inline virtual methods missing the attribute. We - // will return true when we found at least one non-inline virtual with the - // attribute. (This lets our caller know that the attribute needs to be - // propagated up to the vtable.) - return FoundNonInlineVirtualMethodWithAttr; + // Add inrange attribute to indicate that only the VTableIndex can be + // accessed. + unsigned ComponentSize = + CGM.getDataLayout().getTypeAllocSize(CGM.getVTableComponentType()); + unsigned VTableSize = + ComponentSize * Layout.getVTableSize(AddressPoint.VTableIndex); + unsigned Offset = ComponentSize * AddressPoint.AddressPointIndex; + llvm::ConstantRange InRange(llvm::APInt(32, -Offset, true), + llvm::APInt(32, VTableSize - Offset, true)); + return llvm::ConstantExpr::getGetElementPtr( + VTable->getValueType(), VTable, Indices, /*InBounds=*/true, InRange); } llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( @@ -1939,13 +2124,18 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( VirtualPointerIndex); // And load the address point from the VTT. - return CGF.Builder.CreateAlignedLoad(CGF.GlobalsVoidPtrTy, VTT, - CGF.getPointerAlign()); -} + llvm::Value *AP = + CGF.Builder.CreateAlignedLoad(CGF.GlobalsVoidPtrTy, VTT, + CGF.getPointerAlign()); + + if (auto &Schema = CGF.CGM.getCodeGenOpts().PointerAuth.CXXVTTVTablePointers) { + CGPointerAuthInfo PointerAuth = CGF.EmitPointerAuthInfo(Schema, VTT, + GlobalDecl(), + QualType()); + AP = CGF.EmitPointerAuthAuth(PointerAuth, AP); + } -llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - return getVTableAddressPoint(Base, VTableClass); + return AP; } llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, @@ -1980,26 +2170,10 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, getContext().toCharUnitsFromBits(PAlign).getAsAlign()); VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - // In MS C++ if you have a class with virtual functions in which you are using - // selective member import/export, then all virtual functions must be exported - // unless they are inline, otherwise a link error will result. To match this - // behavior, for such classes, we dllimport the vtable if it is defined - // externally and all the non-inline virtual methods are marked dllimport, and - // we dllexport the vtable if it is defined in this TU and all the non-inline - // virtual methods are marked dllexport. - if (CGM.getTarget().hasPS4DLLImportExport()) { - if ((!RD->hasAttr<DLLImportAttr>()) && (!RD->hasAttr<DLLExportAttr>())) { - if (CGM.getVTables().isVTableExternal(RD)) { - if (CXXRecordAllNonInlineVirtualsHaveAttr<DLLImportAttr>(RD)) - VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); - } else { - if (CXXRecordAllNonInlineVirtualsHaveAttr<DLLExportAttr>(RD)) - VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); - } - } - } - CGM.setGVProperties(VTable, RD); + if (CGM.getTarget().hasPS4DLLImportExport()) + setVTableSelectiveDLLImportExport(CGM, VTable, RD); + CGM.setGVProperties(VTable, RD); return VTable; } @@ -2013,8 +2187,9 @@ CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, llvm::Value *VTable = CGF.GetVTablePtr(This, PtrTy, MethodDecl->getParent()); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); - llvm::Value *VFunc; - if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { + llvm::Value *VFunc, *VTableSlotPtr = nullptr; + auto &Schema = CGM.getCodeGenOpts().PointerAuth.CXXVirtualFunctionPointers; + if (!Schema && CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { VFunc = CGF.EmitVTableTypeCheckedLoad( MethodDecl->getParent(), VTable, PtrTy, VTableIndex * @@ -2029,7 +2204,7 @@ CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, CGM.getIntrinsic(llvm::Intrinsic::load_relative, {CGM.Int32Ty}), {VTable, llvm::ConstantInt::get(CGM.Int32Ty, 4 * VTableIndex)}); } else { - llvm::Value *VTableSlotPtr = CGF.Builder.CreateConstInBoundsGEP1_64( + VTableSlotPtr = CGF.Builder.CreateConstInBoundsGEP1_64( PtrTy, VTable, VTableIndex, "vfn"); VFuncLoad = CGF.Builder.CreateAlignedLoad(PtrTy, VTableSlotPtr, CGF.getPointerAlign()); @@ -2053,7 +2228,13 @@ CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, VFunc = VFuncLoad; } - CGCallee Callee(GD, VFunc); + CGPointerAuthInfo PointerAuth; + if (Schema) { + assert(VTableSlotPtr && "virtual function pointer not set"); + GD = CGM.getItaniumVTableContext().findOriginalMethod(GD.getCanonicalDecl()); + PointerAuth = CGF.EmitPointerAuthInfo(Schema, VTableSlotPtr, GD, QualType()); + } + CGCallee Callee(GD, VFunc, PointerAuth); return Callee; } @@ -2079,8 +2260,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( ThisTy = D->getDestroyedType(); } - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr, - QualType(), nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, + nullptr, QualType(), nullptr); return nullptr; } @@ -2134,6 +2315,9 @@ bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const { if (!canSpeculativelyEmitVTableAsBaseClass(RD)) return false; + if (RD->shouldEmitInExternalSource()) + return false; + // For a complete-object vtable (or more specifically, for the VTT), we need // to be able to speculatively emit the vtables of all dynamic virtual bases. for (const auto &B : RD->vbases()) { @@ -2149,11 +2333,12 @@ bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const { } static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, Address InitialPtr, + const CXXRecordDecl *UnadjustedClass, int64_t NonVirtualAdjustment, int64_t VirtualAdjustment, bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) - return InitialPtr.getPointer(); + return InitialPtr.emitRawPointer(CGF); Address V = InitialPtr.withElementType(CGF.Int8Ty); @@ -2166,8 +2351,8 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, // Perform the virtual adjustment if we have one. llvm::Value *ResultPtr; if (VirtualAdjustment) { - Address VTablePtrPtr = V.withElementType(CGF.Int8PtrTy); - llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr); + llvm::Value *VTablePtr = + CGF.GetVTablePtr(V, CGF.Int8PtrTy, UnadjustedClass); llvm::Value *Offset; llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64( @@ -2186,10 +2371,10 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, CGF.getPointerAlign()); } // Adjust our pointer. - ResultPtr = CGF.Builder.CreateInBoundsGEP( - V.getElementType(), V.getPointer(), Offset); + ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getElementType(), + V.emitRawPointer(CGF), Offset); } else { - ResultPtr = V.getPointer(); + ResultPtr = V.emitRawPointer(CGF); } // In a derived-to-base conversion, the non-virtual adjustment is @@ -2202,18 +2387,20 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, return ResultPtr; } -llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, - Address This, - const ThisAdjustment &TA) { - return performTypeAdjustment(CGF, This, TA.NonVirtual, - TA.Virtual.Itanium.VCallOffsetOffset, +llvm::Value * +ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This, + const CXXRecordDecl *UnadjustedClass, + const ThunkInfo &TI) { + return performTypeAdjustment(CGF, This, UnadjustedClass, TI.This.NonVirtual, + TI.This.Virtual.Itanium.VCallOffsetOffset, /*IsReturnAdjustment=*/false); } llvm::Value * ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, + const CXXRecordDecl *UnadjustedClass, const ReturnAdjustment &RA) { - return performTypeAdjustment(CGF, Ret, RA.NonVirtual, + return performTypeAdjustment(CGF, Ret, UnadjustedClass, RA.NonVirtual, RA.Virtual.Itanium.VBaseOffsetOffset, /*IsReturnAdjustment=*/true); } @@ -2275,7 +2462,7 @@ Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie"); - CGF.Builder.CreateCall(F, NumElementsPtr.getPointer()); + CGF.Builder.CreateCall(F, NumElementsPtr.emitRawPointer(CGF)); } // Finally, compute a pointer to the actual data buffer by skipping @@ -2306,7 +2493,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, llvm::FunctionType::get(CGF.SizeTy, CGF.UnqualPtrTy, false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie"); - return CGF.Builder.CreateCall(F, numElementsPtr.getPointer()); + return CGF.Builder.CreateCall(F, numElementsPtr.emitRawPointer(CGF)); } CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) { @@ -2618,7 +2805,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // Call __cxa_guard_release. This cannot throw. CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), - guardAddr.getPointer()); + guardAddr.emitRawPointer(CGF)); } else if (D.isLocalVarDecl()) { // For local variables, store 1 into the first byte of the guard variable // after the object initialization completes so that initialization is @@ -2669,6 +2856,14 @@ static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF, if (llvm::Function *fn = dyn_cast<llvm::Function>(atexit.getCallee())) fn->setDoesNotThrow(); + const auto &Context = CGF.CGM.getContext(); + FunctionProtoType::ExtProtoInfo EPI(Context.getDefaultCallingConvention( + /*IsVariadic=*/false, /*IsCXXMethod=*/false)); + QualType fnType = + Context.getFunctionType(Context.VoidTy, {Context.VoidPtrTy}, EPI); + llvm::Constant *dtorCallee = cast<llvm::Constant>(dtor.getCallee()); + dtorCallee = CGF.CGM.getFunctionPointer(dtorCallee, fnType); + if (!addr) // addr is null when we are trying to register a dtor annotated with // __attribute__((destructor)) in a constructor function. Using null here is @@ -2676,7 +2871,7 @@ static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF, // function. addr = llvm::Constant::getNullValue(CGF.Int8PtrTy); - llvm::Value *args[] = {dtor.getCallee(), addr, handle}; + llvm::Value *args[] = {dtorCallee, addr, handle}; CGF.EmitNounwindRuntimeCall(atexit, args); } @@ -3111,10 +3306,10 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, LValue LV; if (VD->getType()->isReferenceType()) - LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType); + LV = CGF.MakeNaturalAlignRawAddrLValue(CallVal, LValType); else - LV = CGF.MakeAddrLValue(CallVal, LValType, - CGF.getContext().getDeclAlign(VD)); + LV = CGF.MakeRawAddrLValue(CallVal, LValType, + CGF.getContext().getDeclAlign(VD)); // FIXME: need setObjCGCLValueClass? return LV; } @@ -3139,6 +3334,78 @@ bool ItaniumCXXABI::NeedsVTTParameter(GlobalDecl GD) { return false; } +llvm::Constant * +ItaniumCXXABI::getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD) { + SmallString<256> MethodName; + llvm::raw_svector_ostream Out(MethodName); + getMangleContext().mangleCXXName(MD, Out); + MethodName += "_vfpthunk_"; + StringRef ThunkName = MethodName.str(); + llvm::Function *ThunkFn; + if ((ThunkFn = cast_or_null<llvm::Function>( + CGM.getModule().getNamedValue(ThunkName)))) + return ThunkFn; + + const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeCXXMethodDeclaration(MD); + llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(FnInfo); + llvm::GlobalValue::LinkageTypes Linkage = + MD->isExternallyVisible() ? llvm::GlobalValue::LinkOnceODRLinkage + : llvm::GlobalValue::InternalLinkage; + ThunkFn = + llvm::Function::Create(ThunkTy, Linkage, ThunkName, &CGM.getModule()); + if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage) + ThunkFn->setVisibility(llvm::GlobalValue::HiddenVisibility); + assert(ThunkFn->getName() == ThunkName && "name was uniqued!"); + + CGM.SetLLVMFunctionAttributes(MD, FnInfo, ThunkFn, /*IsThunk=*/true); + CGM.SetLLVMFunctionAttributesForDefinition(MD, ThunkFn); + + // Stack protection sometimes gets inserted after the musttail call. + ThunkFn->removeFnAttr(llvm::Attribute::StackProtect); + ThunkFn->removeFnAttr(llvm::Attribute::StackProtectStrong); + ThunkFn->removeFnAttr(llvm::Attribute::StackProtectReq); + + // Start codegen. + CodeGenFunction CGF(CGM); + CGF.CurGD = GlobalDecl(MD); + CGF.CurFuncIsThunk = true; + + // Build FunctionArgs. + FunctionArgList FunctionArgs; + CGF.BuildFunctionArgList(CGF.CurGD, FunctionArgs); + + CGF.StartFunction(GlobalDecl(), FnInfo.getReturnType(), ThunkFn, FnInfo, + FunctionArgs, MD->getLocation(), SourceLocation()); + llvm::Value *ThisVal = loadIncomingCXXThis(CGF); + setCXXABIThisValue(CGF, ThisVal); + + CallArgList CallArgs; + for (const VarDecl *VD : FunctionArgs) + CGF.EmitDelegateCallArg(CallArgs, VD, SourceLocation()); + + const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>(); + RequiredArgs Required = RequiredArgs::forPrototypePlus(FPT, /*this*/ 1); + const CGFunctionInfo &CallInfo = + CGM.getTypes().arrangeCXXMethodCall(CallArgs, FPT, Required, 0); + CGCallee Callee = CGCallee::forVirtual(nullptr, GlobalDecl(MD), + getThisAddress(CGF), ThunkTy); + llvm::CallBase *CallOrInvoke; + CGF.EmitCall(CallInfo, Callee, ReturnValueSlot(), CallArgs, &CallOrInvoke, + /*IsMustTail=*/true, SourceLocation(), true); + auto *Call = cast<llvm::CallInst>(CallOrInvoke); + Call->setTailCallKind(llvm::CallInst::TCK_MustTail); + if (Call->getType()->isVoidTy()) + CGF.Builder.CreateRetVoid(); + else + CGF.Builder.CreateRet(Call); + + // Finish the function to maintain CodeGenFunction invariants. + // FIXME: Don't emit unreachable code. + CGF.EmitBlock(CGF.createBasicBlock()); + CGF.FinishFunction(); + return ThunkFn; +} + namespace { class ItaniumRTTIBuilder { CodeGenModule &CGM; // Per-module state. @@ -3284,7 +3551,7 @@ ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) { // Import the typeinfo symbol when all non-inline virtual methods are // imported. if (CGM.getTarget().hasPS4DLLImportExport()) { - if (RD && CXXRecordAllNonInlineVirtualsHaveAttr<DLLImportAttr>(RD)) { + if (RD && CXXRecordNonInlineHasAttr<DLLImportAttr>(RD)) { GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass); CGM.setDSOLocal(GV); } @@ -3364,6 +3631,8 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) { #include "clang/Basic/RISCVVTypes.def" #define WASM_TYPE(Name, Id, SingletonId) case BuiltinType::Id: #include "clang/Basic/WebAssemblyReferenceTypes.def" +#define AMDGPU_TYPE(Name, Id, SingletonId) case BuiltinType::Id: +#include "clang/Basic/AMDGPUTypes.def" case BuiltinType::ShortAccum: case BuiltinType::Accum: case BuiltinType::LongAccum: @@ -3583,6 +3852,9 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) { case Type::Pipe: llvm_unreachable("Pipe types shouldn't get here"); + case Type::ArrayParameter: + llvm_unreachable("Array Parameter types should not get here."); + case Type::Builtin: case Type::BitInt: // GCC treats vector and complex types as fundamental types. @@ -3691,6 +3963,10 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) { VTable, Two); } + if (auto &Schema = CGM.getCodeGenOpts().PointerAuth.CXXTypeInfoVTablePointer) + VTable = CGM.getConstantSignedPointer(VTable, Schema, nullptr, GlobalDecl(), + QualType(Ty, 0)); + Fields.push_back(VTable); } @@ -3867,6 +4143,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo( case Type::ConstantArray: case Type::IncompleteArray: case Type::VariableArray: + case Type::ArrayParameter: // Itanium C++ ABI 2.9.5p5: // abi::__array_type_info adds no data members to std::type_info. break; @@ -3933,13 +4210,13 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo( // Export the typeinfo in the same circumstances as the vtable is exported. auto GVDLLStorageClass = DLLStorageClass; - if (CGM.getTarget().hasPS4DLLImportExport()) { + if (CGM.getTarget().hasPS4DLLImportExport() && + GVDLLStorageClass != llvm::GlobalVariable::DLLExportStorageClass) { if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) { const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl()); if (RD->hasAttr<DLLExportAttr>() || - CXXRecordAllNonInlineVirtualsHaveAttr<DLLExportAttr>(RD)) { + CXXRecordNonInlineHasAttr<DLLExportAttr>(RD)) GVDLLStorageClass = llvm::GlobalVariable::DLLExportStorageClass; - } } } @@ -3979,9 +4256,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo( CGM.setDSOLocal(GV); TypeName->setDLLStorageClass(DLLStorageClass); - GV->setDLLStorageClass(CGM.getTarget().hasPS4DLLImportExport() - ? GVDLLStorageClass - : DLLStorageClass); + GV->setDLLStorageClass(GVDLLStorageClass); TypeName->setPartition(CGM.getCodeGenOpts().SymbolPartition); GV->setPartition(CGM.getCodeGenOpts().SymbolPartition); @@ -4595,7 +4870,7 @@ static void InitCatchParam(CodeGenFunction &CGF, CGF.Builder.CreateStore(Casted, ExnPtrTmp); // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp.getPointer(); + AdjustedExn = ExnPtrTmp.emitRawPointer(CGF); } } @@ -4821,6 +5096,18 @@ ItaniumCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This, return {CGF.GetVTablePtr(This, CGM.Int8PtrTy, RD), RD}; } +llvm::Constant * +ItaniumCXXABI::getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD) { + const CXXMethodDecl *origMD = + cast<CXXMethodDecl>(CGM.getItaniumVTableContext() + .findOriginalMethod(MD->getCanonicalDecl()) + .getDecl()); + llvm::Constant *thunk = getOrCreateVirtualFunctionPointerThunk(origMD); + QualType funcType = CGM.getContext().getMemberPointerType( + MD->getType(), MD->getParent()->getTypeForDecl()); + return CGM.getMemberFunctionPointer(thunk, funcType); +} + void WebAssemblyCXXABI::emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) { if (CGF.getTarget().hasFeature("exception-handling")) @@ -4869,7 +5156,8 @@ void XLCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, } // Create __dtor function for the var decl. - llvm::Function *DtorStub = CGF.createAtExitStub(D, Dtor, Addr); + llvm::Function *DtorStub = + cast<llvm::Function>(CGF.createAtExitStub(D, Dtor, Addr)); // Register above __dtor with atexit(). CGF.registerGlobalDtorWithAtExit(DtorStub); diff --git a/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.cpp b/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.cpp index 6ce2b94c1db8..44b2df52f001 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.cpp @@ -14,16 +14,20 @@ #include "LinkInModulesPass.h" #include "BackendConsumer.h" +#include "clang/Basic/CodeGenOptions.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/SourceManager.h" + using namespace llvm; -LinkInModulesPass::LinkInModulesPass(clang::BackendConsumer *BC, - bool ShouldLinkFiles) - : BC(BC), ShouldLinkFiles(ShouldLinkFiles) {} +LinkInModulesPass::LinkInModulesPass(clang::BackendConsumer *BC) : BC(BC) {} PreservedAnalyses LinkInModulesPass::run(Module &M, ModuleAnalysisManager &AM) { + if (!BC) + return PreservedAnalyses::all(); - if (BC && BC->LinkInModules(&M, ShouldLinkFiles)) - report_fatal_error("Bitcode module linking failed, compilation aborted!"); + if (BC->LinkInModules(&M)) + report_fatal_error("Bitcode module postopt linking failed, aborted!"); - return PreservedAnalyses::all(); + return PreservedAnalyses::none(); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.h b/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.h index 7fe94d625058..3edbfd076e15 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.h +++ b/contrib/llvm-project/clang/lib/CodeGen/LinkInModulesPass.h @@ -28,10 +28,9 @@ class Pass; /// for use with the legacy pass manager. class LinkInModulesPass : public PassInfoMixin<LinkInModulesPass> { clang::BackendConsumer *BC; - bool ShouldLinkFiles; public: - LinkInModulesPass(clang::BackendConsumer *BC, bool ShouldLinkFiles = true); + LinkInModulesPass(clang::BackendConsumer *BC); PreservedAnalyses run(Module &M, AnalysisManager<Module> &); static bool isRequired() { return true; } diff --git a/contrib/llvm-project/clang/lib/CodeGen/MCDCState.h b/contrib/llvm-project/clang/lib/CodeGen/MCDCState.h new file mode 100644 index 000000000000..e0dd28ff90ed --- /dev/null +++ b/contrib/llvm-project/clang/lib/CodeGen/MCDCState.h @@ -0,0 +1,49 @@ +//===---- MCDCState.h - Per-Function MC/DC state ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Per-Function MC/DC state for PGO +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_CODEGEN_MCDCSTATE_H +#define LLVM_CLANG_LIB_CODEGEN_MCDCSTATE_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ProfileData/Coverage/MCDCTypes.h" + +namespace clang { +class Stmt; +} // namespace clang + +namespace clang::CodeGen::MCDC { + +using namespace llvm::coverage::mcdc; + +/// Per-Function MC/DC state +struct State { + unsigned BitmapBits = 0; + + struct Decision { + unsigned BitmapIdx; + llvm::SmallVector<std::array<int, 2>> Indices; + }; + + llvm::DenseMap<const Stmt *, Decision> DecisionByStmt; + + struct Branch { + ConditionID ID; + const Stmt *DecisionStmt; + }; + + llvm::DenseMap<const Stmt *, Branch> BranchByStmt; +}; + +} // namespace clang::CodeGen::MCDC + +#endif // LLVM_CLANG_LIB_CODEGEN_MCDCSTATE_H diff --git a/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.cpp b/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.cpp index 8589869f6e2f..c5d1e3ad5a20 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.cpp @@ -168,8 +168,8 @@ void MacroPPCallbacks::FileChanged(SourceLocation Loc, FileChangeReason Reason, void MacroPPCallbacks::InclusionDirective( SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName, bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File, - StringRef SearchPath, StringRef RelativePath, const Module *Imported, - SrcMgr::CharacteristicKind FileType) { + StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule, + bool ModuleImported, SrcMgr::CharacteristicKind FileType) { // Record the line location of the current included file. LastHashLoc = HashLoc; diff --git a/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.h b/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.h index 5af177d0c3fa..5f468648da04 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.h +++ b/contrib/llvm-project/clang/lib/CodeGen/MacroPPCallbacks.h @@ -102,7 +102,8 @@ public: StringRef FileName, bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File, StringRef SearchPath, - StringRef RelativePath, const Module *Imported, + StringRef RelativePath, const Module *SuggestedModule, + bool ModuleImported, SrcMgr::CharacteristicKind FileType) override; /// Hook called whenever a macro definition is seen. diff --git a/contrib/llvm-project/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm-project/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 172c4c937b97..76d0191a7e63 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -144,7 +144,7 @@ public: return CatchTypeInfo{nullptr, 0x40}; } - bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; + bool shouldTypeidBeNullChecked(QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, Address ThisPtr, @@ -327,10 +327,6 @@ public: CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase) override; - llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) override; - llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -419,9 +415,11 @@ public: bool exportThunk() override { return false; } llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, - const ThisAdjustment &TA) override; + const CXXRecordDecl * /*UnadjustedClass*/, + const ThunkInfo &TI) override; llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, + const CXXRecordDecl * /*UnadjustedClass*/, const ReturnAdjustment &RA) override; void EmitThreadLocalInitFuncs( @@ -937,7 +935,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, } CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer()); + CPI->setArgOperand(2, var.getObjectAddress(CGF).emitRawPointer(CGF)); CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI); CGF.EmitAutoVarCleanups(var); } @@ -974,18 +972,16 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value, llvm::Value *Offset = GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase); llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP( - Value.getElementType(), Value.getPointer(), Offset); + Value.getElementType(), Value.emitRawPointer(CGF), Offset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase); return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset, PolymorphicBase); } -bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref, - QualType SrcRecordTy) { +bool MicrosoftCXXABI::shouldTypeidBeNullChecked(QualType SrcRecordTy) { const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); - return IsDeref && - !getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); + return !getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); } static llvm::CallBase *emitRTtypeidCall(CodeGenFunction &CGF, @@ -1011,7 +1007,7 @@ llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF, llvm::Type *StdTypeInfoPtrTy) { std::tie(ThisPtr, std::ignore, std::ignore) = performBaseAdjustment(CGF, ThisPtr, SrcRecordTy); - llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer()); + llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.emitRawPointer(CGF)); return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy); } @@ -1033,7 +1029,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastCall( llvm::Value *Offset; std::tie(This, Offset, std::ignore) = performBaseAdjustment(CGF, This, SrcRecordTy); - llvm::Value *ThisPtr = This.getPointer(); + llvm::Value *ThisPtr = This.emitRawPointer(CGF); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); // PVOID __RTDynamicCast( @@ -1065,7 +1061,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction( llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false), "__RTCastToVoid"); - llvm::Value *Args[] = {Value.getPointer()}; + llvm::Value *Args[] = {Value.emitRawPointer(CGF)}; return CGF.EmitRuntimeCall(Function, Args); } @@ -1115,7 +1111,7 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty, const Type *Base = nullptr; uint64_t NumElts = 0; if (CGM.getTarget().getTriple().isAArch64() && - CGM.getTypes().getABIInfo().isHomogeneousAggregate(Ty, Base, NumElts) && + CGM.getABIInfo().isHomogeneousAggregate(Ty, Base, NumElts) && isa<VectorType>(Base)) { return true; } @@ -1126,7 +1122,22 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty, // No base classes // No virtual functions // Additionally, we need to ensure that there is a trivial copy assignment - // operator, a trivial destructor and no user-provided constructors. + // operator, a trivial destructor, no user-provided constructors and no + // deleted copy assignment operator. + + // We need to cover two cases when checking for a deleted copy assignment + // operator. + // + // struct S { int& r; }; + // The above will have an implicit copy assignment operator that is deleted + // and there will not be a `CXXMethodDecl` for the copy assignment operator. + // This is handled by the `needsImplicitCopyAssignment()` check below. + // + // struct S { S& operator=(const S&) = delete; int i; }; + // The above will not have an implicit copy assignment operator that is + // deleted but there is a deleted `CXXMethodDecl` for the declared copy + // assignment operator. This is handled by the `isDeleted()` check below. + if (RD->hasProtectedFields() || RD->hasPrivateFields()) return false; if (RD->getNumBases() > 0) @@ -1135,9 +1146,20 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty, return false; if (RD->hasNonTrivialCopyAssignment()) return false; - for (const CXXConstructorDecl *Ctor : RD->ctors()) - if (Ctor->isUserProvided()) - return false; + if (RD->needsImplicitCopyAssignment() && !RD->hasSimpleCopyAssignment()) + return false; + for (const Decl *D : RD->decls()) { + if (auto *Ctor = dyn_cast<CXXConstructorDecl>(D)) { + if (Ctor->isUserProvided()) + return false; + } else if (auto *Template = dyn_cast<FunctionTemplateDecl>(D)) { + if (isa<CXXConstructorDecl>(Template->getTemplatedDecl())) + return false; + } else if (auto *MethodDecl = dyn_cast<CXXMethodDecl>(D)) { + if (MethodDecl->isCopyAssignmentOperator() && MethodDecl->isDeleted()) + return false; + } + } if (RD->hasNonTrivialDestructor()) return false; return true; @@ -1493,7 +1515,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( llvm::Value *VBaseOffset = GetVirtualBaseClassOffset(CGF, Result, Derived, VBase); llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP( - Result.getElementType(), Result.getPointer(), VBaseOffset); + Result.getElementType(), Result.emitRawPointer(CGF), VBaseOffset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase); Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign); @@ -1660,7 +1682,8 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, llvm::Value *Implicit = getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating); // = nullptr - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, + CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), + ThisTy, /*ImplicitParam=*/Implicit, /*ImplicitParamTy=*/QualType(), nullptr); if (BaseDtorEndBB) { @@ -1791,13 +1814,6 @@ MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base, return VFTablesMap[ID]; } -llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass); - assert(VFTable && "Couldn't find a vftable for the given base?"); - return VFTable; -} - llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { // getAddrOfVTable may return 0 if asked to get an address of a vtable which @@ -2013,8 +2029,9 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( } This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, - ImplicitParam, Context.IntTy, CE); + RValue RV = + CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, + ImplicitParam, Context.IntTy, CE); return RV.getScalarVal(); } @@ -2208,17 +2225,18 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, GV->setLinkage(llvm::GlobalVariable::AvailableExternallyLinkage); } -llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, - Address This, - const ThisAdjustment &TA) { +llvm::Value *MicrosoftCXXABI::performThisAdjustment( + CodeGenFunction &CGF, Address This, + const CXXRecordDecl * /*UnadjustedClass*/, const ThunkInfo &TI) { + const ThisAdjustment &TA = TI.This; if (TA.isEmpty()) - return This.getPointer(); + return This.emitRawPointer(CGF); This = This.withElementType(CGF.Int8Ty); llvm::Value *V; if (TA.Virtual.isEmpty()) { - V = This.getPointer(); + V = This.emitRawPointer(CGF); } else { assert(TA.Virtual.Microsoft.VtordispOffset < 0); // Adjust the this argument based on the vtordisp value. @@ -2227,7 +2245,7 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset)); VtorDispPtr = VtorDispPtr.withElementType(CGF.Int32Ty); llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp"); - V = CGF.Builder.CreateGEP(This.getElementType(), This.getPointer(), + V = CGF.Builder.CreateGEP(This.getElementType(), This.emitRawPointer(CGF), CGF.Builder.CreateNeg(VtorDisp)); // Unfortunately, having applied the vtordisp means that we no @@ -2260,15 +2278,16 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, return V; } -llvm::Value * -MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, - const ReturnAdjustment &RA) { +llvm::Value *MicrosoftCXXABI::performReturnAdjustment( + CodeGenFunction &CGF, Address Ret, + const CXXRecordDecl * /*UnadjustedClass*/, const ReturnAdjustment &RA) { + if (RA.isEmpty()) - return Ret.getPointer(); + return Ret.emitRawPointer(CGF); Ret = Ret.withElementType(CGF.Int8Ty); - llvm::Value *V = Ret.getPointer(); + llvm::Value *V = Ret.emitRawPointer(CGF); if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); int32_t IntSize = CGF.getIntSize().getQuantity(); @@ -2583,7 +2602,7 @@ struct ResetGuardBit final : EHScopeStack::Cleanup { struct CallInitThreadAbort final : EHScopeStack::Cleanup { llvm::Value *Guard; - CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {} + CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {} void Emit(CodeGenFunction &CGF, Flags flags) override { // Calling _Init_thread_abort will reset the guard's state. @@ -3123,8 +3142,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, llvm::Value **VBPtrOut) { CGBuilderTy &Builder = CGF.Builder; // Load the vbtable pointer from the vbptr in the instance. - llvm::Value *VBPtr = Builder.CreateInBoundsGEP(CGM.Int8Ty, This.getPointer(), - VBPtrOffset, "vbptr"); + llvm::Value *VBPtr = Builder.CreateInBoundsGEP( + CGM.Int8Ty, This.emitRawPointer(CGF), VBPtrOffset, "vbptr"); if (VBPtrOut) *VBPtrOut = VBPtr; @@ -3203,7 +3222,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( Builder.CreateBr(SkipAdjustBB); CGF.EmitBlock(SkipAdjustBB); llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base"); - Phi->addIncoming(Base.getPointer(), OriginalBB); + Phi->addIncoming(Base.emitRawPointer(CGF), OriginalBB); Phi->addIncoming(AdjustedBase, VBaseAdjustBB); return Phi; } @@ -3238,7 +3257,7 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - Addr = Base.getPointer(); + Addr = Base.emitRawPointer(CGF); } // Apply the offset, which we assume is non-null. @@ -3526,7 +3545,7 @@ CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - ThisPtrForCall = This.getPointer(); + ThisPtrForCall = This.emitRawPointer(CGF); } if (NonVirtualBaseAdjustment) @@ -4445,10 +4464,7 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { llvm::GlobalVariable *TI = getThrowInfo(ThrowType); // Call into the runtime to throw the exception. - llvm::Value *Args[] = { - AI.getPointer(), - TI - }; + llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI}; CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/ModuleBuilder.cpp b/contrib/llvm-project/clang/lib/CodeGen/ModuleBuilder.cpp index 3594f4c66e67..d4e0ab0339a8 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/ModuleBuilder.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/ModuleBuilder.cpp @@ -180,7 +180,7 @@ namespace { bool HandleTopLevelDecl(DeclGroupRef DG) override { // FIXME: Why not return false and abort parsing? - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return true; HandlingTopLevelDeclRAII HandlingDecl(*this); @@ -206,7 +206,7 @@ namespace { } void HandleInlineFunctionDefinition(FunctionDecl *D) override { - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return; assert(D->doesThisDeclarationHaveABody()); @@ -233,7 +233,7 @@ namespace { /// client hack on the type, which can occur at any point in the file /// (because these can be defined in declspecs). void HandleTagDeclDefinition(TagDecl *D) override { - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return; // Don't allow re-entrant calls to CodeGen triggered by PCH @@ -269,7 +269,7 @@ namespace { } void HandleTagDeclRequiredDefinition(const TagDecl *D) override { - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return; // Don't allow re-entrant calls to CodeGen triggered by PCH @@ -283,7 +283,7 @@ namespace { void HandleTranslationUnit(ASTContext &Ctx) override { // Release the Builder when there is no error. - if (!Diags.hasErrorOccurred() && Builder) + if (!Diags.hasUnrecoverableErrorOccurred() && Builder) Builder->Release(); // If there are errors before or when releasing the Builder, reset @@ -297,25 +297,25 @@ namespace { } void AssignInheritanceModel(CXXRecordDecl *RD) override { - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return; Builder->RefreshTypeCacheForClass(RD); } void CompleteTentativeDefinition(VarDecl *D) override { - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return; Builder->EmitTentativeDefinition(D); } - void CompleteExternalDeclaration(VarDecl *D) override { + void CompleteExternalDeclaration(DeclaratorDecl *D) override { Builder->EmitExternalDeclaration(D); } void HandleVTable(CXXRecordDecl *RD) override { - if (Diags.hasErrorOccurred()) + if (Diags.hasUnrecoverableErrorOccurred()) return; Builder->EmitVTable(RD); diff --git a/contrib/llvm-project/clang/lib/CodeGen/SwiftCallingConv.cpp b/contrib/llvm-project/clang/lib/CodeGen/SwiftCallingConv.cpp index 16fbf52a517d..ab2e2bd0b306 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/SwiftCallingConv.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/SwiftCallingConv.cpp @@ -78,7 +78,7 @@ void SwiftAggLowering::addTypedData(QualType type, CharUnits begin) { QualType eltType = arrayType->getElementType(); auto eltSize = CGM.getContext().getTypeSizeInChars(eltType); - for (uint64_t i = 0, e = arrayType->getSize().getZExtValue(); i != e; ++i) { + for (uint64_t i = 0, e = arrayType->getZExtSize(); i != e; ++i) { addTypedData(eltType, begin + i * eltSize); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.cpp b/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.cpp index 60224d458f6a..64a9a5554caf 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.cpp @@ -19,6 +19,7 @@ #include "clang/CodeGen/CGFunctionInfo.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" +#include "llvm/IR/Function.h" #include "llvm/IR/Type.h" #include "llvm/Support/raw_ostream.h" @@ -206,6 +207,51 @@ llvm::Value *TargetCodeGenInfo::createEnqueuedBlockKernel( return F; } +void TargetCodeGenInfo::setBranchProtectionFnAttributes( + const TargetInfo::BranchProtectionInfo &BPI, llvm::Function &F) { + // Called on already created and initialized function where attributes already + // set from command line attributes but some might need to be removed as the + // actual BPI is different. + if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) { + F.addFnAttr("sign-return-address", BPI.getSignReturnAddrStr()); + F.addFnAttr("sign-return-address-key", BPI.getSignKeyStr()); + } else { + if (F.hasFnAttribute("sign-return-address")) + F.removeFnAttr("sign-return-address"); + if (F.hasFnAttribute("sign-return-address-key")) + F.removeFnAttr("sign-return-address-key"); + } + + auto AddRemoveAttributeAsSet = [&](bool Set, const StringRef &ModAttr) { + if (Set) + F.addFnAttr(ModAttr); + else if (F.hasFnAttribute(ModAttr)) + F.removeFnAttr(ModAttr); + }; + + AddRemoveAttributeAsSet(BPI.BranchTargetEnforcement, + "branch-target-enforcement"); + AddRemoveAttributeAsSet(BPI.BranchProtectionPAuthLR, + "branch-protection-pauth-lr"); + AddRemoveAttributeAsSet(BPI.GuardedControlStack, "guarded-control-stack"); +} + +void TargetCodeGenInfo::initBranchProtectionFnAttributes( + const TargetInfo::BranchProtectionInfo &BPI, llvm::AttrBuilder &FuncAttrs) { + // Only used for initializing attributes in the AttrBuilder, which will not + // contain any of these attributes so no need to remove anything. + if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) { + FuncAttrs.addAttribute("sign-return-address", BPI.getSignReturnAddrStr()); + FuncAttrs.addAttribute("sign-return-address-key", BPI.getSignKeyStr()); + } + if (BPI.BranchTargetEnforcement) + FuncAttrs.addAttribute("branch-target-enforcement"); + if (BPI.BranchProtectionPAuthLR) + FuncAttrs.addAttribute("branch-protection-pauth-lr"); + if (BPI.GuardedControlStack) + FuncAttrs.addAttribute("guarded-control-stack"); +} + namespace { class DefaultTargetCodeGenInfo : public TargetCodeGenInfo { public: diff --git a/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.h b/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.h index 7682f197041c..156b4ff4353b 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.h +++ b/contrib/llvm-project/clang/lib/CodeGen/TargetInfo.h @@ -15,11 +15,12 @@ #define LLVM_CLANG_LIB_CODEGEN_TARGETINFO_H #include "CGBuilder.h" -#include "CodeGenModule.h" #include "CGValue.h" +#include "CodeGenModule.h" #include "clang/AST/Type.h" #include "clang/Basic/LLVM.h" #include "clang/Basic/SyncScope.h" +#include "clang/Basic/TargetInfo.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" @@ -84,12 +85,18 @@ public: /// Provides a convenient hook to handle extra target-specific globals. virtual void emitTargetGlobals(CodeGen::CodeGenModule &CGM) const {} + /// Any further codegen related checks that need to be done on a function + /// signature in a target specific manner. + virtual void checkFunctionABI(CodeGenModule &CGM, + const FunctionDecl *Decl) const {} + /// Any further codegen related checks that need to be done on a function call /// in a target specific manner. virtual void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, const FunctionDecl *Callee, - const CallArgList &Args) const {} + const CallArgList &Args, + QualType ReturnType) const {} /// Determines the size of struct _Unwind_Exception on this platform, /// in 8-bit units. The Itanium ABI defines this as: @@ -290,6 +297,11 @@ public: /// Get the AST address space for alloca. virtual LangAS getASTAllocaAddressSpace() const { return LangAS::Default; } + Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr, + LangAS SrcAddr, LangAS DestAddr, + llvm::Type *DestTy, + bool IsNonNull = false) const; + /// Perform address space cast of an expression of pointer type. /// \param V is the LLVM value to be casted to another address space. /// \param SrcAddr is the language address space of \p V. @@ -402,6 +414,17 @@ public: return nullptr; } + // Set the Branch Protection Attributes of the Function accordingly to the + // BPI. Remove attributes that contradict with current BPI. + static void + setBranchProtectionFnAttributes(const TargetInfo::BranchProtectionInfo &BPI, + llvm::Function &F); + + // Add the Branch Protection Attributes of the FuncAttrs. + static void + initBranchProtectionFnAttributes(const TargetInfo::BranchProtectionInfo &BPI, + llvm::AttrBuilder &FuncAttrs); + protected: static std::string qualifyWindowsLibrary(StringRef Lib); @@ -416,6 +439,8 @@ enum class AArch64ABIKind { AAPCS = 0, DarwinPCS, Win64, + AAPCSSoft, + PAuthTest, }; std::unique_ptr<TargetCodeGenInfo> diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp index ee7f95084d2e..97381f673c28 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp @@ -8,6 +8,9 @@ #include "ABIInfoImpl.h" #include "TargetInfo.h" +#include "clang/AST/Decl.h" +#include "clang/Basic/DiagnosticFrontend.h" +#include "llvm/TargetParser/AArch64TargetParser.h" using namespace clang; using namespace clang::CodeGen; @@ -25,6 +28,8 @@ public: AArch64ABIInfo(CodeGenTypes &CGT, AArch64ABIKind Kind) : ABIInfo(CGT), Kind(Kind) {} + bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; } + private: AArch64ABIKind getABIKind() const { return Kind; } bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; } @@ -50,30 +55,37 @@ private: FI.getCallingConvention()); } - Address EmitDarwinVAArg(Address VAListAddr, QualType Ty, - CodeGenFunction &CGF) const; + RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF, + AggValueSlot Slot) const; - Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty, - CodeGenFunction &CGF) const; + RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF, + AArch64ABIKind Kind, AggValueSlot Slot) const; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override { + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override { llvm::Type *BaseTy = CGF.ConvertType(Ty); if (isa<llvm::ScalableVectorType>(BaseTy)) llvm::report_fatal_error("Passing SVE types to variadic functions is " "currently not supported"); - return Kind == AArch64ABIKind::Win64 ? EmitMSVAArg(CGF, VAListAddr, Ty) - : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF) - : EmitAAPCSVAArg(VAListAddr, Ty, CGF); + return Kind == AArch64ABIKind::Win64 + ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot) + : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot) + : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot); } - Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; bool allowBFloatArgsAndRet() const override { return getTarget().hasBFloat16Type(); } + + using ABIInfo::appendAttributeMangling; + void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index, + raw_ostream &Out) const override; + void appendAttributeMangling(StringRef AttrStr, + raw_ostream &Out) const override; }; class AArch64SwiftABIInfo : public SwiftABIInfo { @@ -108,38 +120,20 @@ public: if (!FD) return; - const auto *TA = FD->getAttr<TargetAttr>(); - if (TA == nullptr) - return; + TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts()); - ParsedTargetAttr Attr = - CGM.getTarget().parseTargetAttr(TA->getFeaturesStr()); - if (Attr.BranchProtection.empty()) - return; - - TargetInfo::BranchProtectionInfo BPI; - StringRef Error; - (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection, - Attr.CPU, BPI, Error); - assert(Error.empty()); - - auto *Fn = cast<llvm::Function>(GV); - static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"}; - Fn->addFnAttr("sign-return-address", SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]); - - if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) { - Fn->addFnAttr("sign-return-address-key", - BPI.SignKey == LangOptions::SignReturnAddressKeyKind::AKey - ? "a_key" - : "b_key"); + if (const auto *TA = FD->getAttr<TargetAttr>()) { + ParsedTargetAttr Attr = + CGM.getTarget().parseTargetAttr(TA->getFeaturesStr()); + if (!Attr.BranchProtection.empty()) { + StringRef Error; + (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection, + Attr.CPU, BPI, Error); + assert(Error.empty()); + } } - - Fn->addFnAttr("branch-target-enforcement", - BPI.BranchTargetEnforcement ? "true" : "false"); - Fn->addFnAttr("branch-protection-pauth-lr", - BPI.BranchProtectionPAuthLR ? "true" : "false"); - Fn->addFnAttr("guarded-control-stack", - BPI.GuardedControlStack ? "true" : "false"); + auto *Fn = cast<llvm::Function>(GV); + setBranchProtectionFnAttributes(BPI, *Fn); } bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF, @@ -155,6 +149,28 @@ public: } return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty); } + + void checkFunctionABI(CodeGenModule &CGM, + const FunctionDecl *Decl) const override; + + void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc, + const FunctionDecl *Caller, + const FunctionDecl *Callee, const CallArgList &Args, + QualType ReturnType) const override; + +private: + // Diagnose calls between functions with incompatible Streaming SVE + // attributes. + void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc, + const FunctionDecl *Caller, + const FunctionDecl *Callee) const; + // Diagnose calls which must pass arguments in floating-point registers when + // the selected target does not have floating-point registers. + void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc, + const FunctionDecl *Caller, + const FunctionDecl *Callee, + const CallArgList &Args, + QualType ReturnType) const; }; class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo { @@ -285,7 +301,7 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic, if (const auto *EIT = Ty->getAs<BitIntType>()) if (EIT->getNumBits() > 128) - return getNaturalAlignIndirect(Ty); + return getNaturalAlignIndirect(Ty, false); return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS() ? ABIArgInfo::getExtend(Ty) @@ -482,6 +498,11 @@ bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize, } bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { + // For the soft-float ABI variant, no types are considered to be homogeneous + // aggregates. + if (Kind == AArch64ABIKind::AAPCSSoft) + return false; + // Homogeneous aggregates for AAPCS64 must have base types of a floating // point type or a short-vector type. This is the same as the 32-bit ABI, // but with the difference that any floating-point type is allowed, @@ -512,18 +533,14 @@ bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() return true; } -Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, + CodeGenFunction &CGF, AArch64ABIKind Kind, + AggValueSlot Slot) const { ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true, CGF.CurFnInfo->getCallingConvention()); // Empty records are ignored for parameter passing purposes. - if (AI.isIgnore()) { - uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8; - CharUnits SlotSize = CharUnits::fromQuantity(PointerSize); - VAListAddr = VAListAddr.withElementType(CGF.Int8PtrTy); - auto *Load = CGF.Builder.CreateLoad(VAListAddr); - return Address(Load, CGF.ConvertTypeForMem(Ty), SlotSize); - } + if (AI.isIgnore()) + return Slot.asRValue(); bool IsIndirect = AI.isIndirect(); @@ -538,7 +555,8 @@ Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, BaseTy = ArrTy->getElementType(); NumRegs = ArrTy->getNumElements(); } - bool IsFPR = BaseTy->isFloatingPointTy() || BaseTy->isVectorTy(); + bool IsFPR = Kind != AArch64ABIKind::AAPCSSoft && + (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy()); // The AArch64 va_list type and handling is specified in the Procedure Call // Standard, section B.4: @@ -711,18 +729,7 @@ Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, // Again, stack arguments may need realignment. In this case both integer and // floating-point ones might be affected. if (!IsIndirect && TyAlign.getQuantity() > 8) { - int Align = TyAlign.getQuantity(); - - OnStackPtr = CGF.Builder.CreatePtrToInt(OnStackPtr, CGF.Int64Ty); - - OnStackPtr = CGF.Builder.CreateAdd( - OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1), - "align_stack"); - OnStackPtr = CGF.Builder.CreateAnd( - OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, -Align), - "align_stack"); - - OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy); + OnStackPtr = emitRoundPointerUpToAlignment(CGF, OnStackPtr, TyAlign); } Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty, std::max(CharUnits::fromQuantity(8), TyAlign)); @@ -761,27 +768,34 @@ Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, OnStackBlock, "vaargs.addr"); if (IsIndirect) - return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy, - TyAlign); - - return ResAddr; + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue( + Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy, + TyAlign), + Ty), + Slot); + + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot); } -Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty, + CodeGenFunction &CGF, + AggValueSlot Slot) const { // The backend's lowering doesn't support va_arg for aggregates or // illegal vector types. Lower VAArg here for these cases and use // the LLVM va_arg instruction for everything else. if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty)) - return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()); + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue( + EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()), Ty), + Slot); uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8; CharUnits SlotSize = CharUnits::fromQuantity(PointerSize); // Empty records are ignored for parameter passing purposes. if (isEmptyRecord(getContext(), Ty, true)) - return Address(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"), - CGF.ConvertTypeForMem(Ty), SlotSize); + return Slot.asRValue(); // The size of the actual thing passed, which might end up just // being a pointer for indirect types. @@ -796,12 +810,12 @@ Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty, IsIndirect = !isHomogeneousAggregate(Ty, Base, Members); } - return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, - TyInfo, SlotSize, /*AllowHigherAlign*/ true); + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, SlotSize, + /*AllowHigherAlign*/ true, Slot); } -Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { bool IsIndirect = false; // Composites larger than 16 bytes are passed by reference. @@ -811,7 +825,136 @@ Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, CGF.getContext().getTypeInfoInChars(Ty), CharUnits::fromQuantity(8), - /*allowHigherAlign*/ false); + /*allowHigherAlign*/ false, Slot); +} + +static bool isStreamingCompatible(const FunctionDecl *F) { + if (const auto *T = F->getType()->getAs<FunctionProtoType>()) + return T->getAArch64SMEAttributes() & + FunctionType::SME_PStateSMCompatibleMask; + return false; +} + +// Report an error if an argument or return value of type Ty would need to be +// passed in a floating-point register. +static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags, + const StringRef ABIName, + const AArch64ABIInfo &ABIInfo, + const QualType &Ty, const NamedDecl *D, + SourceLocation loc) { + const Type *HABase = nullptr; + uint64_t HAMembers = 0; + if (Ty->isFloatingType() || Ty->isVectorType() || + ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) { + Diags.Report(loc, diag::err_target_unsupported_type_for_abi) + << D->getDeclName() << Ty << ABIName; + } +} + +// If we are using a hard-float ABI, but do not have floating point registers, +// then report an error for any function arguments or returns which would be +// passed in floating-pint registers. +void AArch64TargetCodeGenInfo::checkFunctionABI( + CodeGenModule &CGM, const FunctionDecl *FuncDecl) const { + const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>(); + const TargetInfo &TI = ABIInfo.getContext().getTargetInfo(); + + if (!TI.hasFeature("fp") && !ABIInfo.isSoftFloat()) { + diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, + FuncDecl->getReturnType(), FuncDecl, + FuncDecl->getLocation()); + for (ParmVarDecl *PVD : FuncDecl->parameters()) { + diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(), + PVD, FuncDecl->getLocation()); + } + } +} + +void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming( + CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, + const FunctionDecl *Callee) const { + if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>()) + return; + + bool CallerIsStreaming = + IsArmStreamingFunction(Caller, /*IncludeLocallyStreaming=*/true); + bool CalleeIsStreaming = + IsArmStreamingFunction(Callee, /*IncludeLocallyStreaming=*/true); + bool CallerIsStreamingCompatible = isStreamingCompatible(Caller); + bool CalleeIsStreamingCompatible = isStreamingCompatible(Callee); + + if (!CalleeIsStreamingCompatible && + (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) + CGM.getDiags().Report( + CallLoc, CalleeIsStreaming + ? diag::err_function_always_inline_attribute_mismatch + : diag::warn_function_always_inline_attribute_mismatch) + << Caller->getDeclName() << Callee->getDeclName() << "streaming"; + if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) + if (NewAttr->isNewZA()) + CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za) + << Callee->getDeclName(); +} + +// If the target does not have floating-point registers, but we are using a +// hard-float ABI, there is no way to pass floating-point, vector or HFA values +// to functions, so we report an error. +void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat( + CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, + const FunctionDecl *Callee, const CallArgList &Args, + QualType ReturnType) const { + const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>(); + const TargetInfo &TI = ABIInfo.getContext().getTargetInfo(); + + if (!Caller || TI.hasFeature("fp") || ABIInfo.isSoftFloat()) + return; + + diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType, + Callee ? Callee : Caller, CallLoc); + + for (const CallArg &Arg : Args) + diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(), + Callee ? Callee : Caller, CallLoc); +} + +void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM, + SourceLocation CallLoc, + const FunctionDecl *Caller, + const FunctionDecl *Callee, + const CallArgList &Args, + QualType ReturnType) const { + checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee); + checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType); +} + +void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr, + unsigned Index, + raw_ostream &Out) const { + appendAttributeMangling(Attr->getFeatureStr(Index), Out); +} + +void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr, + raw_ostream &Out) const { + if (AttrStr == "default") { + Out << ".default"; + return; + } + + Out << "._"; + SmallVector<StringRef, 8> Features; + AttrStr.split(Features, "+"); + for (auto &Feat : Features) + Feat = Feat.trim(); + + llvm::sort(Features, [](const StringRef LHS, const StringRef RHS) { + return LHS.compare(RHS) < 0; + }); + + llvm::SmallDenseSet<StringRef, 8> UniqueFeats; + for (auto &Feat : Features) + if (auto Ext = llvm::AArch64::parseFMVExtension(Feat)) + if (UniqueFeats.insert(Ext->Name).second) + Out << 'M' << Ext->Name; } std::unique_ptr<TargetCodeGenInfo> diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp index 03ac6b78598f..4d3275e17c38 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp @@ -45,11 +45,12 @@ public: ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyKernelArgumentType(QualType Ty) const; - ABIArgInfo classifyArgumentType(QualType Ty, unsigned &NumRegsLeft) const; + ABIArgInfo classifyArgumentType(QualType Ty, bool Variadic, + unsigned &NumRegsLeft) const; void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; bool AMDGPUABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { @@ -103,19 +104,27 @@ void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const { if (!getCXXABI().classifyReturnType(FI)) FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + unsigned ArgumentIndex = 0; + const unsigned numFixedArguments = FI.getNumRequiredArgs(); + unsigned NumRegsLeft = MaxNumRegsForArgsRet; for (auto &Arg : FI.arguments()) { if (CC == llvm::CallingConv::AMDGPU_KERNEL) { Arg.info = classifyKernelArgumentType(Arg.type); } else { - Arg.info = classifyArgumentType(Arg.type, NumRegsLeft); + bool FixedArgument = ArgumentIndex++ < numFixedArguments; + Arg.info = classifyArgumentType(Arg.type, !FixedArgument, NumRegsLeft); } } } -Address AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { - llvm_unreachable("AMDGPU does not support varargs"); +RValue AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { + const bool IsIndirect = false; + const bool AllowHigherAlign = false; + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, + getContext().getTypeInfoInChars(Ty), + CharUnits::fromQuantity(4), AllowHigherAlign, Slot); } ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const { @@ -197,12 +206,20 @@ ABIArgInfo AMDGPUABIInfo::classifyKernelArgumentType(QualType Ty) const { return ABIArgInfo::getDirect(LTy, 0, nullptr, false); } -ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty, +ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty, bool Variadic, unsigned &NumRegsLeft) const { assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow"); Ty = useFirstFieldIfTransparentUnion(Ty); + if (Variadic) { + return ABIArgInfo::getDirect(/*T=*/nullptr, + /*Offset=*/0, + /*Padding=*/nullptr, + /*CanBeFlattened=*/false, + /*Align=*/0); + } + if (isAggregateTypeForABI(Ty)) { // Records with non-trivial destructors/copy-constructors should not be // passed by value. @@ -356,6 +373,29 @@ void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes( if (NumVGPR != 0) F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR)); } + + if (const auto *Attr = FD->getAttr<AMDGPUMaxNumWorkGroupsAttr>()) { + uint32_t X = Attr->getMaxNumWorkGroupsX() + ->EvaluateKnownConstInt(M.getContext()) + .getExtValue(); + // Y and Z dimensions default to 1 if not specified + uint32_t Y = Attr->getMaxNumWorkGroupsY() + ? Attr->getMaxNumWorkGroupsY() + ->EvaluateKnownConstInt(M.getContext()) + .getExtValue() + : 1; + uint32_t Z = Attr->getMaxNumWorkGroupsZ() + ? Attr->getMaxNumWorkGroupsZ() + ->EvaluateKnownConstInt(M.getContext()) + .getExtValue() + : 1; + + llvm::SmallString<32> AttrVal; + llvm::raw_svector_ostream OS(AttrVal); + OS << X << ',' << Y << ',' << Z; + + F->addFnAttr("amdgpu-max-num-workgroups", AttrVal.str()); + } } /// Emits control constants used to change per-architecture behaviour in the diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/ARC.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/ARC.cpp index 550eb4068f25..1904e8fdb388 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/ARC.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/ARC.cpp @@ -24,8 +24,8 @@ public: using DefaultABIInfo::DefaultABIInfo; private: - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; void updateState(const ABIArgInfo &Info, QualType Ty, CCState &State) const { if (!State.FreeRegs) @@ -81,11 +81,11 @@ ABIArgInfo ARCABIInfo::getIndirectByValue(QualType Ty) const { TypeAlign > MinABIStackAlignInBytes); } -Address ARCABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue ARCABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, getContext().getTypeInfoInChars(Ty), - CharUnits::fromQuantity(4), true); + CharUnits::fromQuantity(4), true, Slot); } ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty, diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/ARM.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/ARM.cpp index d7d175ff1724..457d761039a0 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/ARM.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/ARM.cpp @@ -35,7 +35,9 @@ public: case llvm::Triple::EABI: case llvm::Triple::EABIHF: case llvm::Triple::GNUEABI: + case llvm::Triple::GNUEABIT64: case llvm::Triple::GNUEABIHF: + case llvm::Triple::GNUEABIHFT64: case llvm::Triple::MuslEABI: case llvm::Triple::MuslEABIHF: return true; @@ -48,6 +50,7 @@ public: switch (getTarget().getTriple().getEnvironment()) { case llvm::Triple::EABIHF: case llvm::Triple::GNUEABIHF: + case llvm::Triple::GNUEABIHFT64: case llvm::Triple::MuslEABIHF: return true; default: @@ -81,8 +84,8 @@ private: void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; llvm::CallingConv::ID getLLVMDefaultCC() const; llvm::CallingConv::ID getABIDefaultCC() const; @@ -141,7 +144,7 @@ public: ParsedTargetAttr Attr = CGM.getTarget().parseTargetAttr(TA->getFeaturesStr()); if (!Attr.BranchProtection.empty()) { - TargetInfo::BranchProtectionInfo BPI; + TargetInfo::BranchProtectionInfo BPI{}; StringRef DiagMsg; StringRef Arch = Attr.CPU.empty() ? CGM.getTarget().getTargetOpts().CPU : Attr.CPU; @@ -151,17 +154,8 @@ public: D->getLocation(), diag::warn_target_unsupported_branch_protection_attribute) << Arch; - } else { - static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"}; - assert(static_cast<unsigned>(BPI.SignReturnAddr) <= 2 && - "Unexpected SignReturnAddressScopeKind"); - Fn->addFnAttr( - "sign-return-address", - SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]); - - Fn->addFnAttr("branch-target-enforcement", - BPI.BranchTargetEnforcement ? "true" : "false"); - } + } else + setBranchProtectionFnAttributes(BPI, (*Fn)); } else if (CGM.getLangOpts().BranchTargetEnforcement || CGM.getLangOpts().hasSignReturnAddress()) { // If the Branch Protection attribute is missing, validate the target @@ -173,6 +167,10 @@ public: diag::warn_target_unsupported_branch_protection_attribute) << Attr.CPU; } + } else if (CGM.getTarget().isBranchProtectionSupportedArch( + CGM.getTarget().getTargetOpts().CPU)) { + TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts()); + setBranchProtectionFnAttributes(BPI, (*Fn)); } const ARMInterruptAttr *Attr = FD->getAttr<ARMInterruptAttr>(); @@ -677,7 +675,7 @@ bool ARMABIInfo::isIllegalVectorType(QualType Ty) const { /// Return true if a type contains any 16-bit floating point vectors bool ARMABIInfo::containsAnyFP16Vectors(QualType Ty) const { if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) { - uint64_t NElements = AT->getSize().getZExtValue(); + uint64_t NElements = AT->getZExtSize(); if (NElements == 0) return false; return containsAnyFP16Vectors(AT->getElementType()); @@ -759,16 +757,13 @@ bool ARMABIInfo::isEffectivelyAAPCS_VFP(unsigned callConvention, (acceptHalf && (getABIKind() == ARMABIKind::AAPCS16_VFP)); } -Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { CharUnits SlotSize = CharUnits::fromQuantity(4); // Empty records are ignored for parameter passing purposes. - if (isEmptyRecord(getContext(), Ty, true)) { - VAListAddr = VAListAddr.withElementType(CGF.Int8PtrTy); - auto *Load = CGF.Builder.CreateLoad(VAListAddr); - return Address(Load, CGF.ConvertTypeForMem(Ty), SlotSize); - } + if (isEmptyRecord(getContext(), Ty, true)) + return Slot.asRValue(); CharUnits TySize = getContext().getTypeSizeInChars(Ty); CharUnits TyAlignForABI = getContext().getTypeUnadjustedAlignInChars(Ty); @@ -804,8 +799,8 @@ Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, } TypeInfoChars TyInfo(TySize, TyAlignForABI, AlignRequirementKind::None); - return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, - SlotSize, /*AllowHigherAlign*/ true); + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, SlotSize, + /*AllowHigherAlign*/ true, Slot); } std::unique_ptr<TargetCodeGenInfo> diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/CSKY.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/CSKY.cpp index 924eced700e1..d8720afd1a71 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/CSKY.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/CSKY.cpp @@ -33,8 +33,8 @@ public: bool isReturnType = false) const; ABIArgInfo classifyReturnType(QualType RetTy) const; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; } // end anonymous namespace @@ -57,20 +57,18 @@ void CSKYABIInfo::computeInfo(CGFunctionInfo &FI) const { } } -Address CSKYABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue CSKYABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { CharUnits SlotSize = CharUnits::fromQuantity(XLen / 8); // Empty records are ignored for parameter passing purposes. - if (isEmptyRecord(getContext(), Ty, true)) { - return Address(CGF.Builder.CreateLoad(VAListAddr), - CGF.ConvertTypeForMem(Ty), SlotSize); - } + if (isEmptyRecord(getContext(), Ty, true)) + return Slot.asRValue(); auto TInfo = getContext().getTypeInfoInChars(Ty); return emitVoidPtrVAArg(CGF, VAListAddr, Ty, false, TInfo, SlotSize, - /*AllowHigherAlign=*/true); + /*AllowHigherAlign=*/true, Slot); } ABIArgInfo CSKYABIInfo::classifyArgumentType(QualType Ty, int &ArgGPRsLeft, diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/Hexagon.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/Hexagon.cpp index 944a8d002ecf..8fd2a81494d9 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/Hexagon.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/Hexagon.cpp @@ -29,8 +29,8 @@ private: void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; Address EmitVAArgFromMemory(CodeGenFunction &CFG, Address VAListAddr, QualType Ty) const; Address EmitVAArgForHexagon(CodeGenFunction &CFG, Address VAListAddr, @@ -408,13 +408,16 @@ Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF, return Address(ArgAddr, MemTy, CharUnits::fromQuantity(ArgAlign)); } -Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { if (getTarget().getTriple().isMusl()) - return EmitVAArgForHexagonLinux(CGF, VAListAddr, Ty); + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue(EmitVAArgForHexagonLinux(CGF, VAListAddr, Ty), Ty), + Slot); - return EmitVAArgForHexagon(CGF, VAListAddr, Ty); + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue(EmitVAArgForHexagon(CGF, VAListAddr, Ty), Ty), Slot); } std::unique_ptr<TargetCodeGenInfo> diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp index 63b9a1fdb988..6af9375461f0 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp @@ -44,8 +44,8 @@ public: int &FARsLeft) const; ABIArgInfo classifyReturnType(QualType RetTy) const; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; ABIArgInfo extendType(QualType Ty) const; @@ -146,7 +146,7 @@ bool LoongArchABIInfo::detectFARsEligibleStructHelper( } if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) { - uint64_t ArraySize = ATy->getSize().getZExtValue(); + uint64_t ArraySize = ATy->getZExtSize(); QualType EltTy = ATy->getElementType(); // Non-zero-length arrays of empty records make the struct ineligible to be // passed via FARs in C++. @@ -417,14 +417,13 @@ ABIArgInfo LoongArchABIInfo::classifyReturnType(QualType RetTy) const { return classifyArgumentType(RetTy, /*IsFixed=*/true, GARsLeft, FARsLeft); } -Address LoongArchABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue LoongArchABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { CharUnits SlotSize = CharUnits::fromQuantity(GRLen / 8); // Empty records are ignored for parameter passing purposes. if (isEmptyRecord(getContext(), Ty, true)) - return Address(CGF.Builder.CreateLoad(VAListAddr), - CGF.ConvertTypeForMem(Ty), SlotSize); + return Slot.asRValue(); auto TInfo = getContext().getTypeInfoInChars(Ty); @@ -432,7 +431,7 @@ Address LoongArchABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*IsIndirect=*/TInfo.Width > 2 * SlotSize, TInfo, SlotSize, - /*AllowHigherAlign=*/true); + /*AllowHigherAlign=*/true, Slot); } ABIArgInfo LoongArchABIInfo::extendType(QualType Ty) const { diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp index bb67d97f4421..8ce70e2111cc 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp @@ -51,9 +51,12 @@ public: I.info = classifyArgumentType(I.type); } - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override { - return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty)); + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override { + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue( + EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty)), Ty), + Slot); } }; diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/Mips.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/Mips.cpp index 8f11c63dcd85..06d9b6d4a576 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/Mips.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/Mips.cpp @@ -34,8 +34,8 @@ public: ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType RetTy, uint64_t &Offset) const; void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; ABIArgInfo extendType(QualType Ty) const; }; @@ -346,8 +346,8 @@ void MipsABIInfo::computeInfo(CGFunctionInfo &FI) const { I.info = classifyArgumentType(I.type, Offset); } -Address MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType OrigTy) const { +RValue MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType OrigTy, AggValueSlot Slot) const { QualType Ty = OrigTy; // Integer arguments are promoted to 32-bit on O32 and 64-bit on N32/N64. @@ -373,28 +373,25 @@ Address MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // MinABIStackAlignInBytes is the size of argument slots on the stack. CharUnits ArgSlotSize = CharUnits::fromQuantity(MinABIStackAlignInBytes); - Address Addr = emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, - TyInfo, ArgSlotSize, /*AllowHigherAlign*/ true); + RValue Res = emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, TyInfo, + ArgSlotSize, /*AllowHigherAlign*/ true, Slot); - - // If there was a promotion, "unpromote" into a temporary. + // If there was a promotion, "unpromote". // TODO: can we just use a pointer into a subset of the original slot? if (DidPromote) { - Address Temp = CGF.CreateMemTemp(OrigTy, "vaarg.promotion-temp"); - llvm::Value *Promoted = CGF.Builder.CreateLoad(Addr); + llvm::Type *ValTy = CGF.ConvertType(OrigTy); + llvm::Value *Promoted = Res.getScalarVal(); // Truncate down to the right width. - llvm::Type *IntTy = (OrigTy->isIntegerType() ? Temp.getElementType() - : CGF.IntPtrTy); + llvm::Type *IntTy = (OrigTy->isIntegerType() ? ValTy : CGF.IntPtrTy); llvm::Value *V = CGF.Builder.CreateTrunc(Promoted, IntTy); if (OrigTy->isPointerType()) - V = CGF.Builder.CreateIntToPtr(V, Temp.getElementType()); + V = CGF.Builder.CreateIntToPtr(V, ValTy); - CGF.Builder.CreateStore(V, Temp); - Addr = Temp; + return RValue::get(V); } - return Addr; + return Res; } ABIArgInfo MipsABIInfo::extendType(QualType Ty) const { diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp index d0dc7c258a03..ec7f1c439b18 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp @@ -32,8 +32,8 @@ public: ABIArgInfo classifyArgumentType(QualType Ty) const; void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; bool isUnsupportedType(QualType T) const; ABIArgInfo coerceToIntArrayWithLimit(QualType Ty, unsigned MaxSize) const; }; @@ -47,6 +47,10 @@ public: CodeGen::CodeGenModule &M) const override; bool shouldEmitStaticExternCAliases() const override; + llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM, + llvm::PointerType *T, + QualType QT) const override; + llvm::Type *getCUDADeviceBuiltinSurfaceDeviceType() const override { // On the device side, surface reference is represented as an object handle // in 64-bit integer. @@ -81,7 +85,7 @@ private: LValue Src) { llvm::Value *Handle = nullptr; llvm::Constant *C = - llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer()); + llvm::dyn_cast<llvm::Constant>(Src.getAddress().emitRawPointer(CGF)); // Lookup `addrspacecast` through the constant pointer if any. if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C)) C = llvm::cast<llvm::Constant>(ASC->getPointerOperand()); @@ -199,8 +203,11 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const { void NVPTXABIInfo::computeInfo(CGFunctionInfo &FI) const { if (!getCXXABI().classifyReturnType(FI)) FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); - for (auto &I : FI.arguments()) - I.info = classifyArgumentType(I.type); + + for (auto &&[ArgumentsCount, I] : llvm::enumerate(FI.arguments())) + I.info = ArgumentsCount < FI.getNumRequiredArgs() + ? classifyArgumentType(I.type) + : ABIArgInfo::getDirect(); // Always honor user-specified calling convention. if (FI.getCallingConvention() != llvm::CallingConv::C) @@ -209,9 +216,12 @@ void NVPTXABIInfo::computeInfo(CGFunctionInfo &FI) const { FI.setEffectiveCallingConvention(getRuntimeCC()); } -Address NVPTXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { - llvm_unreachable("NVPTX does not support varargs"); +RValue NVPTXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*IsIndirect=*/false, + getContext().getTypeInfoInChars(Ty), + CharUnits::fromQuantity(1), + /*AllowHigherAlign=*/true, Slot); } void NVPTXTargetCodeGenInfo::setTargetAttributes( @@ -285,6 +295,20 @@ void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::GlobalValue *GV, bool NVPTXTargetCodeGenInfo::shouldEmitStaticExternCAliases() const { return false; } + +llvm::Constant * +NVPTXTargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM, + llvm::PointerType *PT, + QualType QT) const { + auto &Ctx = CGM.getContext(); + if (PT->getAddressSpace() != Ctx.getTargetAddressSpace(LangAS::opencl_local)) + return llvm::ConstantPointerNull::get(PT); + + auto NPT = llvm::PointerType::get( + PT->getContext(), Ctx.getTargetAddressSpace(LangAS::opencl_generic)); + return llvm::ConstantExpr::getAddrSpaceCast( + llvm::ConstantPointerNull::get(NPT), PT); +} } void CodeGenModule::handleCUDALaunchBoundsAttr(llvm::Function *F, diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/PNaCl.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/PNaCl.cpp index 771aa7469da2..9b7d757df3a3 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/PNaCl.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/PNaCl.cpp @@ -27,8 +27,8 @@ class PNaClABIInfo : public ABIInfo { ABIArgInfo classifyArgumentType(QualType RetTy) const; void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, - Address VAListAddr, QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class PNaClTargetCodeGenInfo : public TargetCodeGenInfo { @@ -45,15 +45,18 @@ void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const { I.info = classifyArgumentType(I.type); } -Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { // The PNaCL ABI is a bit odd, in that varargs don't use normal // function classification. Structs get passed directly for varargs // functions, through a rewriting transform in // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows // this target to actually support a va_arg instructions with an // aggregate type, unlike other targets. - return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()); + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue( + EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()), Ty), + Slot); } /// Classify argument of given type \p Ty. diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/PPC.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/PPC.cpp index 40dddde508c1..e4155810963e 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/PPC.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/PPC.cpp @@ -8,13 +8,14 @@ #include "ABIInfoImpl.h" #include "TargetInfo.h" +#include "clang/Basic/DiagnosticFrontend.h" using namespace clang; using namespace clang::CodeGen; -static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty, CharUnits SlotSize, - CharUnits EltSize, const ComplexType *CTy) { +static RValue complexTempStructure(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, CharUnits SlotSize, + CharUnits EltSize, const ComplexType *CTy) { Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2, SlotSize, SlotSize, /*AllowHigher*/ true); @@ -36,10 +37,7 @@ static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr, llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal"); llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag"); - Address Temp = CGF.CreateMemTemp(Ty, "vacplx"); - CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty), - /*init*/ true); - return Temp; + return RValue::getComplex(Real, Imag); } static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, @@ -128,8 +126,8 @@ public: I.info = classifyArgumentType(I.type); } - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class AIXTargetCodeGenInfo : public TargetCodeGenInfo { @@ -145,6 +143,9 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; + + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &M) const override; }; } // namespace @@ -235,8 +236,8 @@ CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const { return CharUnits::fromQuantity(PtrByteSize); } -Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { auto TypeInfo = getContext().getTypeInfoInChars(Ty); TypeInfo.Align = getParamTypeAlignment(Ty); @@ -257,7 +258,7 @@ Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, } return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo, - SlotSize, /*AllowHigher*/ true); + SlotSize, /*AllowHigher*/ true, Slot); } bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable( @@ -265,6 +266,61 @@ bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable( return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true); } +void AIXTargetCodeGenInfo::setTargetAttributes( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { + if (!isa<llvm::GlobalVariable>(GV)) + return; + + auto *GVar = cast<llvm::GlobalVariable>(GV); + auto GVId = GV->getName(); + + // Is this a global variable specified by the user as toc-data? + bool UserSpecifiedTOC = + llvm::binary_search(M.getCodeGenOpts().TocDataVarsUserSpecified, GVId); + // Assumes the same variable cannot be in both TocVarsUserSpecified and + // NoTocVars. + if (UserSpecifiedTOC || + ((M.getCodeGenOpts().AllTocData) && + !llvm::binary_search(M.getCodeGenOpts().NoTocDataVars, GVId))) { + const unsigned long PointerSize = + GV->getParent()->getDataLayout().getPointerSizeInBits() / 8; + auto *VarD = dyn_cast<VarDecl>(D); + assert(VarD && "Invalid declaration of global variable."); + + ASTContext &Context = D->getASTContext(); + unsigned Alignment = Context.toBits(Context.getDeclAlign(D)) / 8; + const auto *Ty = VarD->getType().getTypePtr(); + const RecordDecl *RDecl = + Ty->isRecordType() ? Ty->getAs<RecordType>()->getDecl() : nullptr; + + bool EmitDiagnostic = UserSpecifiedTOC && GV->hasExternalLinkage(); + auto reportUnsupportedWarning = [&](bool ShouldEmitWarning, StringRef Msg) { + if (ShouldEmitWarning) + M.getDiags().Report(D->getLocation(), diag::warn_toc_unsupported_type) + << GVId << Msg; + }; + if (!Ty || Ty->isIncompleteType()) + reportUnsupportedWarning(EmitDiagnostic, "of incomplete type"); + else if (RDecl && RDecl->hasFlexibleArrayMember()) + reportUnsupportedWarning(EmitDiagnostic, + "it contains a flexible array member"); + else if (VarD->getTLSKind() != VarDecl::TLS_None) + reportUnsupportedWarning(EmitDiagnostic, "of thread local storage"); + else if (PointerSize < Context.getTypeInfo(VarD->getType()).Width / 8) + reportUnsupportedWarning(EmitDiagnostic, + "variable is larger than a pointer"); + else if (PointerSize < Alignment) + reportUnsupportedWarning(EmitDiagnostic, + "variable is aligned wider than a pointer"); + else if (D->hasAttr<SectionAttr>()) + reportUnsupportedWarning(EmitDiagnostic, + "variable has a section attribute"); + else if (GV->hasExternalLinkage() || + (M.getCodeGenOpts().AllTocData && !GV->hasLocalLinkage())) + GVar->addAttribute("toc-data"); + } +} + // PowerPC-32 namespace { /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. @@ -289,8 +345,8 @@ public: I.info = classifyArgumentType(I.type); } - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class PPC32TargetCodeGenInfo : public TargetCodeGenInfo { @@ -367,8 +423,8 @@ ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { // TODO: this implementation is now likely redundant with // DefaultABIInfo::EmitVAArg. -Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, - QualType Ty) const { +RValue PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, + QualType Ty, AggValueSlot Slot) const { if (getTarget().getTriple().isOSDarwin()) { auto TI = getContext().getTypeInfoInChars(Ty); TI.Align = getParamTypeAlignment(Ty); @@ -376,14 +432,14 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, CharUnits SlotSize = CharUnits::fromQuantity(4); return emitVoidPtrVAArg(CGF, VAList, Ty, classifyArgumentType(Ty).isIndirect(), TI, SlotSize, - /*AllowHigherAlign=*/true); + /*AllowHigherAlign=*/true, Slot); } const unsigned OverflowLimit = 8; if (const ComplexType *CTy = Ty->getAs<ComplexType>()) { // TODO: Implement this. For now ignore. (void)CTy; - return Address::invalid(); // FIXME? + return RValue::getAggregate(Address::invalid()); // FIXME? } // struct __va_list_tag { @@ -454,9 +510,10 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8); llvm::Value *RegOffset = Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity())); - RegAddr = Address( - Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset), - DirectTy, RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); + RegAddr = Address(Builder.CreateInBoundsGEP( + CGF.Int8Ty, RegAddr.emitRawPointer(CGF), RegOffset), + DirectTy, + RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); // Increase the used-register count. NumRegs = @@ -492,7 +549,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Round up address of argument to alignment CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty); if (Align > OverflowAreaAlign) { - llvm::Value *Ptr = OverflowArea.getPointer(); + llvm::Value *Ptr = OverflowArea.emitRawPointer(CGF); OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align), OverflowArea.getElementType(), Align); } @@ -501,7 +558,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Increase the overflow area. OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size); - Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr); + Builder.CreateStore(OverflowArea.emitRawPointer(CGF), OverflowAreaAddr); CGF.EmitBranch(Cont); } @@ -517,7 +574,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, getContext().getTypeAlignInChars(Ty)); } - return Result; + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(Result, Ty), Slot); } bool PPC32TargetCodeGenInfo::isStructReturnInRegABI( @@ -598,8 +655,8 @@ public: } } - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo { @@ -898,8 +955,8 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { } // Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine. -Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { auto TypeInfo = getContext().getTypeInfoInChars(Ty); TypeInfo.Align = getParamTypeAlignment(Ty); @@ -931,7 +988,7 @@ Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // types this way, and so right-alignment only applies to fundamental types. // So on PPC64, we must force the use of right-alignment even for aggregates. return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo, - SlotSize, /*AllowHigher*/ true, + SlotSize, /*AllowHigher*/ true, Slot, /*ForceRightAdjust*/ true); } diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/RISCV.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/RISCV.cpp index 02c86ad2e58c..f2add9351c03 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/RISCV.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/RISCV.cpp @@ -48,8 +48,8 @@ public: int &ArgFPRsLeft) const; ABIArgInfo classifyReturnType(QualType RetTy) const; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; ABIArgInfo extendType(QualType Ty) const; @@ -152,7 +152,7 @@ bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff, } if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) { - uint64_t ArraySize = ATy->getSize().getZExtValue(); + uint64_t ArraySize = ATy->getZExtSize(); QualType EltTy = ATy->getElementType(); // Non-zero-length arrays of empty records make the struct ineligible for // the FP calling convention in C++. @@ -361,12 +361,13 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed, CGCXXABI::RAA_DirectInMemory); } - // Ignore empty structs/unions. - if (isEmptyRecord(getContext(), Ty, true)) - return ABIArgInfo::getIgnore(); - uint64_t Size = getContext().getTypeSize(Ty); + // Ignore empty structs/unions whose size is zero. According to the calling + // convention empty structs/unions are required to be sized types in C++. + if (isEmptyRecord(getContext(), Ty, true) && Size == 0) + return ABIArgInfo::getIgnore(); + // Pass floating point values via FPRs if possible. if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() && FLen >= Size && ArgFPRsLeft) { @@ -441,7 +442,13 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed, return getNaturalAlignIndirect(Ty, /*ByVal=*/false); } - return ABIArgInfo::getDirect(); + ABIArgInfo Info = ABIArgInfo::getDirect(); + + // If it is tuple type, it can't be flattened. + if (llvm::StructType *STy = dyn_cast<llvm::StructType>(CGT.ConvertType(Ty))) + Info.setCanBeFlattened(!STy->containsHomogeneousScalableVectorTypes()); + + return Info; } if (const VectorType *VT = Ty->getAs<VectorType>()) @@ -483,15 +490,13 @@ ABIArgInfo RISCVABIInfo::classifyReturnType(QualType RetTy) const { ArgFPRsLeft); } -Address RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { CharUnits SlotSize = CharUnits::fromQuantity(XLen / 8); // Empty records are ignored for parameter passing purposes. - if (isEmptyRecord(getContext(), Ty, true)) { - return Address(CGF.Builder.CreateLoad(VAListAddr), - CGF.ConvertTypeForMem(Ty), SlotSize); - } + if (isEmptyRecord(getContext(), Ty, true)) + return Slot.asRValue(); auto TInfo = getContext().getTypeInfoInChars(Ty); @@ -505,8 +510,8 @@ Address RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Arguments bigger than 2*Xlen bytes are passed indirectly. bool IsIndirect = TInfo.Width > 2 * SlotSize; - return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TInfo, - SlotSize, /*AllowHigherAlign=*/true); + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TInfo, SlotSize, + /*AllowHigherAlign=*/true, Slot); } ABIArgInfo RISCVABIInfo::extendType(QualType Ty) const { @@ -523,7 +528,10 @@ public: RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen, bool EABI) : TargetCodeGenInfo( - std::make_unique<RISCVABIInfo>(CGT, XLen, FLen, EABI)) {} + std::make_unique<RISCVABIInfo>(CGT, XLen, FLen, EABI)) { + SwiftInfo = + std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false); + } void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/Sparc.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/Sparc.cpp index a337a52a94ec..da8c7219be26 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/Sparc.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/Sparc.cpp @@ -111,8 +111,8 @@ public: private: ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit) const; void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; // Coercion type builder for structs passed in registers. The coercion type // serves two purposes: @@ -263,7 +263,11 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { CoerceBuilder CB(getVMContext(), getDataLayout()); CB.addStruct(0, StrTy); - CB.pad(llvm::alignTo(CB.DL.getTypeSizeInBits(StrTy), 64)); + // All structs, even empty ones, should take up a register argument slot, + // so pin the minimum struct size to one bit. + CB.pad(llvm::alignTo( + std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), uint64_t(1)), + 64)); // Try to use the original type for coercion. llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType(); @@ -274,8 +278,8 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { return ABIArgInfo::getDirect(CoerceTy); } -Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { ABIArgInfo AI = classifyType(Ty, 16 * 8); llvm::Type *ArgTy = CGT.ConvertType(Ty); if (AI.canHaveCoerceToType() && !AI.getCoerceToType()) @@ -321,14 +325,15 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, break; case ABIArgInfo::Ignore: - return Address(llvm::UndefValue::get(ArgPtrTy), ArgTy, TypeInfo.Align); + return Slot.asRValue(); } // Update VAList. Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next"); - Builder.CreateStore(NextPtr.getPointer(), VAListAddr); + Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); - return ArgAddr.withElementType(ArgTy); + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue(ArgAddr.withElementType(ArgTy), Ty), Slot); } void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const { diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/SystemZ.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/SystemZ.cpp index 6eb0c6ef2f7d..4d61f5137934 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/SystemZ.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/SystemZ.cpp @@ -38,8 +38,8 @@ public: ABIArgInfo classifyArgumentType(QualType ArgTy) const; void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class SystemZTargetCodeGenInfo : public TargetCodeGenInfo { @@ -243,8 +243,8 @@ QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const { return Ty; } -Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { // Assume that va_list type is correct; should be pointer to LLVM type: // struct { // i64 __gpr; @@ -306,11 +306,11 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update overflow_arg_area_ptr pointer llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( - OverflowArgArea.getElementType(), OverflowArgArea.getPointer(), + OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); - return MemAddr; + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(MemAddr, Ty), Slot); } assert(PaddedSize.getQuantity() == 8); @@ -382,10 +382,9 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, Address MemAddr = RawMemAddr.withElementType(DirectTy); // Update overflow_arg_area_ptr pointer - llvm::Value *NewOverflowArgArea = - CGF.Builder.CreateGEP(OverflowArgArea.getElementType(), - OverflowArgArea.getPointer(), PaddedSizeV, - "overflow_arg_area"); + llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( + OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), + PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); CGF.EmitBranch(ContBlock); @@ -398,7 +397,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"), ArgTy, TyInfo.Align); - return ResAddr; + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot); } ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { @@ -413,13 +412,16 @@ ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { } ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { + // Handle transparent union types. + Ty = useFirstFieldIfTransparentUnion(Ty); + // Handle the generic C++ ABI. if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); // Integers and enums are extended to full register width. if (isPromotableIntegerTypeForABI(Ty)) - return ABIArgInfo::getExtend(Ty); + return ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty)); // Handle vector types and vector-like structure types. Note that // as opposed to float-like structure types, we do not allow any diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/WebAssembly.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/WebAssembly.cpp index bd332228ce5b..70a968fe93ca 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/WebAssembly.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/WebAssembly.cpp @@ -41,8 +41,8 @@ private: Arg.info = classifyArgumentType(Arg.type); } - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo { @@ -155,15 +155,15 @@ ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const { return defaultInfo.classifyReturnType(RetTy); } -Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { bool IsIndirect = isAggregateTypeForABI(Ty) && !isEmptyRecord(getContext(), Ty, true) && !isSingleElementStruct(Ty, getContext()); return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, getContext().getTypeInfoInChars(Ty), CharUnits::fromQuantity(4), - /*AllowHigherAlign=*/true); + /*AllowHigherAlign=*/true, Slot); } std::unique_ptr<TargetCodeGenInfo> diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/X86.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/X86.cpp index 2291c991fb11..1dc3172a6bdf 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/X86.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/X86.cpp @@ -173,8 +173,8 @@ class X86_32ABIInfo : public ABIInfo { public: void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI, @@ -327,7 +327,7 @@ void X86_32TargetCodeGenInfo::addReturnRegisterOutputs( ResultTruncRegTypes.push_back(CoerceTy); // Coerce the integer by bitcasting the return slot pointer. - ReturnSlot.setAddress(ReturnSlot.getAddress(CGF).withElementType(CoerceTy)); + ReturnSlot.setAddress(ReturnSlot.getAddress().withElementType(CoerceTy)); ResultRegDests.push_back(ReturnSlot); rewriteInputConstraintReferences(NumOutputs, 1, AsmString); @@ -469,7 +469,8 @@ bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const { ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const { // If the return value is indirect, then the hidden argument is consuming one // integer register. - if (State.FreeRegs) { + if (State.CC != llvm::CallingConv::X86_FastCall && + State.CC != llvm::CallingConv::X86_VectorCall && State.FreeRegs) { --State.FreeRegs; if (!IsMCUABI) return getNaturalAlignIndirectInReg(RetTy); @@ -792,6 +793,8 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State, return ABIArgInfo::getDirect(); return ABIArgInfo::getExpand(); } + if (IsVectorCall && Ty->isBuiltinType()) + return ABIArgInfo::getDirect(); return getIndirectResult(Ty, /*ByVal=*/false, State); } @@ -1064,11 +1067,17 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { StackAlign); } -Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, - Address VAListAddr, QualType Ty) const { +RValue X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { auto TypeInfo = getContext().getTypeInfoInChars(Ty); + CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo)); + ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0); + // Empty records are ignored for parameter passing purposes. + if (AI.isIgnore()) + return Slot.asRValue(); + // x86-32 changes the alignment of certain arguments on the stack. // // Just messing with TypeInfo like this works because we never pass @@ -1076,9 +1085,9 @@ Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, TypeInfo.Align = CharUnits::fromQuantity( getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity())); - return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, - TypeInfo, CharUnits::fromQuantity(4), - /*AllowHigherAlign*/ true); + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo, + CharUnits::fromQuantity(4), + /*AllowHigherAlign*/ true, Slot); } bool X86_32TargetCodeGenInfo::isStructReturnInRegABI( @@ -1359,10 +1368,10 @@ public: void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; - Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; + RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; bool has64BitPointers() const { return Has64BitPointers; @@ -1378,8 +1387,8 @@ public: void computeInfo(CGFunctionInfo &FI) const override; - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; bool isHomogeneousAggregateBaseType(QualType Ty) const override { // FIXME: Assumes vectorcall is in use. @@ -1476,8 +1485,8 @@ public: void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, - const FunctionDecl *Callee, - const CallArgList &Args) const override; + const FunctionDecl *Callee, const CallArgList &Args, + QualType ReturnType) const override; }; } // namespace @@ -1552,9 +1561,15 @@ static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx, return false; } -void X86_64TargetCodeGenInfo::checkFunctionCallABI( - CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, - const FunctionDecl *Callee, const CallArgList &Args) const { +void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM, + SourceLocation CallLoc, + const FunctionDecl *Caller, + const FunctionDecl *Callee, + const CallArgList &Args, + QualType ReturnType) const { + if (!Callee) + return; + llvm::StringMap<bool> CallerMap; llvm::StringMap<bool> CalleeMap; unsigned ArgIndex = 0; @@ -1993,7 +2008,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo, // this, but it isn't worth it and would be harder to verify. Current = NoClass; uint64_t EltSize = getContext().getTypeSize(AT->getElementType()); - uint64_t ArraySize = AT->getSize().getZExtValue(); + uint64_t ArraySize = AT->getZExtSize(); // The only case a 256-bit wide vector could be used is when the array // contains a single 256-bit element. Since Lo and Hi logic isn't extended @@ -2081,7 +2096,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo, bool BitField = i->isBitField(); // Ignore padding bit-fields. - if (BitField && i->isUnnamedBitfield()) + if (BitField && i->isUnnamedBitField()) continue; // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than @@ -2100,8 +2115,11 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo, postMerge(Size, Lo, Hi); return; } + + bool IsInMemory = + Offset % getContext().getTypeAlign(i->getType().getCanonicalType()); // Note, skip this test for bit-fields, see below. - if (!BitField && Offset % getContext().getTypeAlign(i->getType())) { + if (!BitField && IsInMemory) { Lo = Memory; postMerge(Size, Lo, Hi); return; @@ -2119,7 +2137,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo, // structure to be passed in memory even if unaligned, and // therefore they can straddle an eightbyte. if (BitField) { - assert(!i->isUnnamedBitfield()); + assert(!i->isUnnamedBitField()); uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx); uint64_t Size = i->getBitWidthValue(getContext()); @@ -2295,7 +2313,7 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit, if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) { unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType()); - unsigned NumElts = (unsigned)AT->getSize().getZExtValue(); + unsigned NumElts = (unsigned)AT->getZExtSize(); // Check each element to see if the element overlaps with the queried range. for (unsigned i = 0; i != NumElts; ++i) { @@ -2788,12 +2806,11 @@ X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs, // memory), except in situations involving unions. case X87Up: case SSE: + ++neededSSE; HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8); if (Lo == NoClass) // Pass HighPart at offset 8 in memory. return ABIArgInfo::getDirect(HighPart, 8); - - ++neededSSE; break; // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the @@ -3004,8 +3021,8 @@ static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF, return Address(Res, LTy, Align); } -Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { // Assume that va_list type is correct; should be pointer to LLVM type: // struct { // i32 gp_offset; @@ -3019,10 +3036,16 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE, /*isNamedArg*/false); + // Empty records are ignored for parameter passing purposes. + if (AI.isIgnore()) + return Slot.asRValue(); + // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed // in the registers. If not go to step 7. if (!neededInt && !neededSSE) - return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty); + return CGF.EmitLoadOfAnyValue( + CGF.MakeAddrLValue(EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty), Ty), + Slot); // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of // general purpose registers needed to pass type and num_fp to hold @@ -3185,11 +3208,11 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, CGF.EmitBlock(ContBlock); Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock, "vaarg.addr"); - return ResAddr; + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot); } -Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." uint64_t Width = getContext().getTypeSize(Ty); @@ -3198,7 +3221,7 @@ Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, CGF.getContext().getTypeInfoInChars(Ty), CharUnits::fromQuantity(8), - /*allowHigherAlign*/ false); + /*allowHigherAlign*/ false, Slot); } ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall( @@ -3390,8 +3413,8 @@ void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { } } -Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." uint64_t Width = getContext().getTypeSize(Ty); @@ -3400,7 +3423,7 @@ Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, CGF.getContext().getTypeInfoInChars(Ty), CharUnits::fromQuantity(8), - /*allowHigherAlign*/ false); + /*allowHigherAlign*/ false, Slot); } std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo( diff --git a/contrib/llvm-project/clang/lib/CodeGen/Targets/XCore.cpp b/contrib/llvm-project/clang/lib/CodeGen/Targets/XCore.cpp index aeb48f851e16..f3e241171b87 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/Targets/XCore.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/Targets/XCore.cpp @@ -113,8 +113,8 @@ public: class XCoreABIInfo : public DefaultABIInfo { public: XCoreABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} - Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const override; + RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, + AggValueSlot Slot) const override; }; class XCoreTargetCodeGenInfo : public TargetCodeGenInfo { @@ -134,8 +134,8 @@ public: // TODO: this implementation is likely now redundant with the default // EmitVAArg. -Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, - QualType Ty) const { +RValue XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty, AggValueSlot Slot) const { CGBuilderTy &Builder = CGF.Builder; // Get the VAList. @@ -180,10 +180,10 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Increment the VAList. if (!ArgSize.isZero()) { Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize); - Builder.CreateStore(APN.getPointer(), VAListAddr); + Builder.CreateStore(APN.emitRawPointer(CGF), VAListAddr); } - return Val; + return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(Val, Ty), Slot); } /// During the expansion of a RecordType, an incomplete TypeString is placed |