diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp | 318 |
1 files changed, 211 insertions, 107 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp index b6c2567bd578..8e0604181fb1 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp @@ -66,7 +66,7 @@ Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, const Twine &Name, llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); - Alloca->setAlignment(Align.getQuantity()); + Alloca->setAlignment(Align.getAsAlign()); return Address(Alloca, Align); } @@ -126,7 +126,7 @@ Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) { assert(isa<llvm::AllocaInst>(Var.getPointer())); auto *Store = new llvm::StoreInst(Init, Var.getPointer()); - Store->setAlignment(Var.getAlignment().getQuantity()); + Store->setAlignment(Var.getAlignment().getAsAlign()); llvm::BasicBlock *Block = AllocaInsertPt->getParent(); Block->getInstList().insertAfter(AllocaInsertPt->getIterator(), Store); } @@ -392,7 +392,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, llvm::GlobalValue::NotThreadLocal, CGF.getContext().getTargetAddressSpace(AS)); CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); - GV->setAlignment(alignment.getQuantity()); + GV->setAlignment(alignment.getAsAlign()); llvm::Constant *C = GV; if (AS != LangAS::Default) C = TCG.performAddrSpaceCast( @@ -417,7 +417,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, LValue CodeGenFunction:: EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { - const Expr *E = M->GetTemporaryExpr(); + const Expr *E = M->getSubExpr(); assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) || !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) && @@ -516,13 +516,13 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end // marker. Instead, start the lifetime of a conditional temporary earlier - // so that it's unconditional. Don't do this in ASan's use-after-scope - // mode so that it gets the more precise lifetime marks. If the type has - // a non-trivial destructor, we'll have a cleanup block for it anyway, - // so this typically doesn't help; skip it in that case. + // so that it's unconditional. Don't do this with sanitizers which need + // more precise lifetime marks. ConditionalEvaluation *OldConditional = nullptr; CGBuilderTy::InsertPoint OldIP; if (isInConditionalBranch() && !E->getType().isDestructedType() && + !SanOpts.has(SanitizerKind::HWAddress) && + !SanOpts.has(SanitizerKind::Memory) && !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) { OldConditional = OutermostConditional; OutermostConditional = nullptr; @@ -573,7 +573,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { LV = EmitLValueForField(LV, Adjustment.Field); assert(LV.isSimple() && "materialized temporary field is not a simple lvalue"); - Object = LV.getAddress(); + Object = LV.getAddress(*this); break; } @@ -594,7 +594,7 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) { // Emit the expression as an lvalue. LValue LV = EmitLValue(E); assert(LV.isSimple()); - llvm::Value *Value = LV.getPointer(); + llvm::Value *Value = LV.getPointer(*this); if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) { // C++11 [dcl.ref]p5 (as amended by core issue 453): @@ -677,8 +677,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, // Quickly determine whether we have a pointer to an alloca. It's possible // to skip null checks, and some alignment checks, for these pointers. This // can reduce compile-time significantly. - auto PtrToAlloca = - dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCastsNoFollowAliases()); + auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts()); llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext()); llvm::Value *IsNonNull = nullptr; @@ -998,7 +997,7 @@ EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, // Add the inc/dec to the real part. NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec"); } else { - QualType ElemTy = E->getType()->getAs<ComplexType>()->getElementType(); + QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType(); llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1); if (!isInc) FVal.changeSign(); @@ -1012,6 +1011,9 @@ EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, // Store the updated result through the lvalue. EmitStoreOfComplex(IncVal, LV, /*init*/ false); + if (getLangOpts().OpenMP) + CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this, + E->getSubExpr()); // If this is a postinc, return the value read from memory, otherwise use the // updated value. @@ -1130,7 +1132,7 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, LValue LV = EmitLValue(UO->getSubExpr()); if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); - return LV.getAddress(); + return LV.getAddress(*this); } } @@ -1220,8 +1222,8 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase())) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(), - E->getType(), LV.getAlignment(), SkippedChecks); + EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), + LV.getAlignment(), SkippedChecks); } return LV; } @@ -1268,6 +1270,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { case Expr::CXXOperatorCallExprClass: case Expr::UserDefinedLiteralClass: return EmitCallExprLValue(cast<CallExpr>(E)); + case Expr::CXXRewrittenBinaryOperatorClass: + return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm()); case Expr::VAArgExprClass: return EmitVAArgExprLValue(cast<VAArgExpr>(E)); case Expr::DeclRefExprClass: @@ -1306,7 +1310,7 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { if (LV.isSimple()) { // Defend against branches out of gnu statement expressions surrounded by // cleanups. - llvm::Value *V = LV.getPointer(); + llvm::Value *V = LV.getPointer(*this); Scope.ForceCleanup({&V}); return LValue::MakeAddr(Address(V, LV.getAlignment()), LV.getType(), getContext(), LV.getBaseInfo(), LV.getTBAAInfo()); @@ -1522,7 +1526,7 @@ llvm::Value *CodeGenFunction::emitScalarConstant( llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue, SourceLocation Loc) { - return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(), + return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(), lvalue.getType(), Loc, lvalue.getBaseInfo(), lvalue.getTBAAInfo(), lvalue.isNontemporal()); } @@ -1762,7 +1766,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, bool isInit) { - EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(), + EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(), lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal()); } @@ -1773,18 +1777,18 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { if (LV.isObjCWeak()) { // load of a __weak object. - Address AddrWeakObj = LV.getAddress(); + Address AddrWeakObj = LV.getAddress(*this); return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this, AddrWeakObj)); } if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { // In MRC mode, we do a load+autorelease. if (!getLangOpts().ObjCAutoRefCount) { - return RValue::get(EmitARCLoadWeak(LV.getAddress())); + return RValue::get(EmitARCLoadWeak(LV.getAddress(*this))); } // In ARC mode, we load retained and then consume the value. - llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress()); + llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this)); Object = EmitObjCConsumeObject(LV.getType(), Object); return RValue::get(Object); } @@ -1879,8 +1883,7 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) { /// Generates lvalue for partial ext_vector access. Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) { Address VectorAddress = LV.getExtVectorAddress(); - const VectorType *ExprVT = LV.getType()->getAs<VectorType>(); - QualType EQT = ExprVT->getElementType(); + QualType EQT = LV.getType()->castAs<VectorType>()->getElementType(); llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT); Address CastToPointerElement = @@ -1970,9 +1973,10 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, case Qualifiers::OCL_Weak: if (isInit) // Initialize and then skip the primitive store. - EmitARCInitWeak(Dst.getAddress(), Src.getScalarVal()); + EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal()); else - EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true); + EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(), + /*ignore*/ true); return; case Qualifiers::OCL_Autoreleasing: @@ -1985,7 +1989,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (Dst.isObjCWeak() && !Dst.isNonGC()) { // load of a __weak object. - Address LvalueDst = Dst.getAddress(); + Address LvalueDst = Dst.getAddress(*this); llvm::Value *src = Src.getScalarVal(); CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst); return; @@ -1993,7 +1997,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (Dst.isObjCStrong() && !Dst.isNonGC()) { // load of a __strong object. - Address LvalueDst = Dst.getAddress(); + Address LvalueDst = Dst.getAddress(*this); llvm::Value *src = Src.getScalarVal(); if (Dst.isObjCIvar()) { assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); @@ -2195,7 +2199,7 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E, // If ivar is a structure pointer, assigning to field of // this struct follows gcc's behavior and makes it a non-ivar // writer-barrier conservatively. - ExpTy = ExpTy->getAs<PointerType>()->getPointeeType(); + ExpTy = ExpTy->castAs<PointerType>()->getPointeeType(); if (ExpTy->isRecordType()) { LV.setObjCIvar(false); return; @@ -2231,7 +2235,7 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E, // a non-ivar write-barrier. QualType ExpTy = E->getType(); if (ExpTy->isPointerType()) - ExpTy = ExpTy->getAs<PointerType>()->getPointeeType(); + ExpTy = ExpTy->castAs<PointerType>()->getPointeeType(); if (ExpTy->isRecordType()) LV.setObjCIvar(false); } @@ -2319,8 +2323,8 @@ Address CodeGenFunction::EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo, TBAAAccessInfo *PointeeTBAAInfo) { - llvm::LoadInst *Load = Builder.CreateLoad(RefLVal.getAddress(), - RefLVal.isVolatile()); + llvm::LoadInst *Load = + Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); CharUnits Align = getNaturalTypeAlignment(RefLVal.getType()->getPointeeType(), @@ -2362,7 +2366,7 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, // If it's thread_local, emit a call to its wrapper function instead. if (VD->getTLSKind() == VarDecl::TLS_Dynamic && - CGF.CGM.getCXXABI().usesThreadWrapperFunction()) + CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD)) return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T); // Check if the variable is marked as declare target with link clause in // device codegen. @@ -2564,21 +2568,35 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { VD = VD->getCanonicalDecl(); if (auto *FD = LambdaCaptureFields.lookup(VD)) return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue); - else if (CapturedStmtInfo) { + if (CapturedStmtInfo) { auto I = LocalDeclMap.find(VD); if (I != LocalDeclMap.end()) { + LValue CapLVal; if (VD->getType()->isReferenceType()) - return EmitLoadOfReferenceLValue(I->second, VD->getType(), - AlignmentSource::Decl); - return MakeAddrLValue(I->second, T); + CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(), + AlignmentSource::Decl); + else + CapLVal = MakeAddrLValue(I->second, T); + // Mark lvalue as nontemporal if the variable is marked as nontemporal + // in simd context. + if (getLangOpts().OpenMP && + CGM.getOpenMPRuntime().isNontemporalDecl(VD)) + CapLVal.setNontemporal(/*Value=*/true); + return CapLVal; } LValue CapLVal = EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), CapturedStmtInfo->getContextValue()); - return MakeAddrLValue( - Address(CapLVal.getPointer(), getContext().getDeclAlign(VD)), + CapLVal = MakeAddrLValue( + Address(CapLVal.getPointer(*this), getContext().getDeclAlign(VD)), CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), CapLVal.getTBAAInfo()); + // Mark lvalue as nontemporal if the variable is marked as nontemporal + // in simd context. + if (getLangOpts().OpenMP && + CGM.getOpenMPRuntime().isNontemporalDecl(VD)) + CapLVal.setNontemporal(/*Value=*/true); + return CapLVal; } assert(isa<BlockDecl>(CurCodeDecl)); @@ -2711,7 +2729,7 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { // __real is valid on scalars. This is a faster way of testing that. // __imag can only produce an rvalue on scalars. if (E->getOpcode() == UO_Real && - !LV.getAddress().getElementType()->isStructTy()) { + !LV.getAddress(*this).getElementType()->isStructTy()) { assert(E->getSubExpr()->getType()->isArithmeticType()); return LV; } @@ -2719,9 +2737,9 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { QualType T = ExprTy->castAs<ComplexType>()->getElementType(); Address Component = - (E->getOpcode() == UO_Real - ? emitAddrOfRealComponent(LV.getAddress(), LV.getType()) - : emitAddrOfImagComponent(LV.getAddress(), LV.getType())); + (E->getOpcode() == UO_Real + ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType()) + : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType())); LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(), CGM.getTBAAInfoForSubobject(LV, T)); ElemLV.getQuals().addQualifiers(LV.getQuals()); @@ -3199,6 +3217,9 @@ void CodeGenFunction::EmitCfiCheckFail() { llvm::Function *F = llvm::Function::Create( llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false), llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule()); + + CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F); + CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F); F->setVisibility(llvm::GlobalValue::HiddenVisibility); StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args, @@ -3318,7 +3339,7 @@ Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E, // Expressions of array type can't be bitfields or vector elements. LValue LV = EmitLValue(E); - Address Addr = LV.getAddress(); + Address Addr = LV.getAddress(*this); // If the array type was an incomplete type, we need to make sure // the decay ends up being the right type. @@ -3401,10 +3422,48 @@ static QualType getFixedSizeElementType(const ASTContext &ctx, return eltType; } +/// Given an array base, check whether its member access belongs to a record +/// with preserve_access_index attribute or not. +static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) { + if (!ArrayBase || !CGF.getDebugInfo()) + return false; + + // Only support base as either a MemberExpr or DeclRefExpr. + // DeclRefExpr to cover cases like: + // struct s { int a; int b[10]; }; + // struct s *p; + // p[1].a + // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr. + // p->b[5] is a MemberExpr example. + const Expr *E = ArrayBase->IgnoreImpCasts(); + if (const auto *ME = dyn_cast<MemberExpr>(E)) + return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>(); + + if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) { + const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl()); + if (!VarDef) + return false; + + const auto *PtrT = VarDef->getType()->getAs<PointerType>(); + if (!PtrT) + return false; + + const auto *PointeeT = PtrT->getPointeeType() + ->getUnqualifiedDesugaredType(); + if (const auto *RecT = dyn_cast<RecordType>(PointeeT)) + return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>(); + return false; + } + + return false; +} + static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, ArrayRef<llvm::Value *> indices, QualType eltType, bool inbounds, bool signedIndices, SourceLocation loc, + QualType *arrayType = nullptr, + const Expr *Base = nullptr, const llvm::Twine &name = "arrayidx") { // All the indices except that last must be zero. #ifndef NDEBUG @@ -3426,16 +3485,21 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, llvm::Value *eltPtr; auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back()); - if (!CGF.IsInPreservedAIRegion || !LastIndex) { + if (!LastIndex || + (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { eltPtr = emitArraySubscriptGEP( CGF, addr.getPointer(), indices, inbounds, signedIndices, loc, name); } else { // Remember the original array subscript for bpf target unsigned idx = LastIndex->getZExtValue(); - eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getPointer(), + llvm::DIType *DbgInfo = nullptr; + if (arrayType) + DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc); + eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(), + addr.getPointer(), indices.size() - 1, - idx); + idx, DbgInfo); } return Address(eltPtr, eltAlign); @@ -3478,8 +3542,9 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, LValue LHS = EmitLValue(E->getBase()); auto *Idx = EmitIdxAfterBase(/*Promote*/false); assert(LHS.isSimple() && "Can only subscript lvalue vectors here!"); - return LValue::MakeVectorElt(LHS.getAddress(), Idx, E->getBase()->getType(), - LHS.getBaseInfo(), TBAAAccessInfo()); + return LValue::MakeVectorElt(LHS.getAddress(*this), Idx, + E->getBase()->getType(), LHS.getBaseInfo(), + TBAAAccessInfo()); } // All the other cases basically behave like simple offsetting. @@ -3572,19 +3637,22 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, auto *Idx = EmitIdxAfterBase(/*Promote*/true); // Propagate the alignment from the array itself to the result. + QualType arrayType = Array->getType(); Addr = emitArraySubscriptGEP( - *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx}, + *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx}, E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices, - E->getExprLoc()); + E->getExprLoc(), &arrayType, E->getBase()); EltBaseInfo = ArrayLV.getBaseInfo(); EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType()); } else { // The base must be a pointer; emit it with an estimate of its alignment. Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo); auto *Idx = EmitIdxAfterBase(/*Promote*/true); + QualType ptrType = E->getBase()->getType(); Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(), !getLangOpts().isSignedOverflowDefined(), - SignedIndices, E->getExprLoc()); + SignedIndices, E->getExprLoc(), &ptrType, + E->getBase()); } LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo); @@ -3606,7 +3674,7 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) { BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound); if (BaseTy->isArrayType()) { - Address Addr = BaseLVal.getAddress(); + Address Addr = BaseLVal.getAddress(CGF); BaseInfo = BaseLVal.getBaseInfo(); // If the array type was an incomplete type, we need to make sure @@ -3631,7 +3699,7 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, &TypeTBAAInfo); BaseInfo.mergeForCast(TypeBaseInfo); TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo); - return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress()), Align); + return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)), Align); } return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); } @@ -3772,7 +3840,7 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, // Propagate the alignment from the array itself to the result. EltPtr = emitArraySubscriptGEP( - *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx}, + *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx}, ResultExprTy, !getLangOpts().isSignedOverflowDefined(), /*signedIndices=*/false, E->getExprLoc()); BaseInfo = ArrayLV.getBaseInfo(); @@ -3801,7 +3869,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo); - const PointerType *PT = E->getBase()->getType()->getAs<PointerType>(); + const auto *PT = E->getBase()->getType()->castAs<PointerType>(); Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo); Base.getQuals().removeObjCGCAttr(); } else if (E->getBase()->isGLValue()) { @@ -3832,7 +3900,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { if (Base.isSimple()) { llvm::Constant *CV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); - return LValue::MakeExtVectorElt(Base.getAddress(), CV, type, + return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type, Base.getBaseInfo(), TBAAAccessInfo()); } assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!"); @@ -3877,6 +3945,15 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { if (auto *Field = dyn_cast<FieldDecl>(ND)) { LValue LV = EmitLValueForField(BaseLV, Field); setObjCGCLValueClass(getContext(), E, LV); + if (getLangOpts().OpenMP) { + // If the member was explicitly marked as nontemporal, mark it as + // nontemporal. If the base lvalue is marked as nontemporal, mark access + // to children as nontemporal too. + if ((IsWrappedCXXThis(BaseExpr) && + CGM.getOpenMPRuntime().isNontemporalDecl(Field)) || + BaseLV.isNontemporal()) + LV.setNontemporal(/*Value=*/true); + } return LV; } @@ -3983,11 +4060,22 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(field->getParent()); const CGBitFieldInfo &Info = RL.getBitFieldInfo(field); - Address Addr = base.getAddress(); + Address Addr = base.getAddress(*this); unsigned Idx = RL.getLLVMFieldNo(field); - if (Idx != 0) - // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + const RecordDecl *rec = field->getParent(); + if (!IsInPreservedAIRegion && + (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) { + if (Idx != 0) + // For structs, we GEP to the field that the record layout suggests. + Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + } else { + llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( + getContext().getRecordType(rec), rec->getLocation()); + Addr = Builder.CreatePreserveStructAccessIndex(Addr, Idx, + getDebugInfoFIndex(rec, field->getFieldIndex()), + DbgInfo); + } + // Get the access type. llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize); @@ -4040,7 +4128,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, getContext().getTypeSizeInChars(FieldType).getQuantity(); } - Address addr = base.getAddress(); + Address addr = base.getAddress(*this); if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) { if (CGM.getCodeGenOpts().StrictVTablePointers && ClassDef->isDynamicClass()) { @@ -4056,7 +4144,6 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, unsigned RecordCVR = base.getVRQualifiers(); if (rec->isUnion()) { // For unions, there is no pointer adjustment. - assert(!FieldType->isReferenceType() && "union has reference member"); if (CGM.getCodeGenOpts().StrictVTablePointers && hasAnyVptr(FieldType, getContext())) // Because unions can easily skip invariant.barriers, we need to add @@ -4064,7 +4151,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, addr = Address(Builder.CreateLaunderInvariantGroup(addr.getPointer()), addr.getAlignment()); - if (IsInPreservedAIRegion) { + if (IsInPreservedAIRegion || + (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) { // Remember the original union field index llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( getContext().getRecordType(rec), rec->getLocation()); @@ -4073,27 +4161,31 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), addr.getAlignment()); } - } else { - if (!IsInPreservedAIRegion) + if (FieldType->isReferenceType()) + addr = Builder.CreateElementBitCast( + addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName()); + } else { + if (!IsInPreservedAIRegion && + (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) // For structs, we GEP to the field that the record layout suggests. addr = emitAddrOfFieldStorage(*this, addr, field); else // Remember the original struct field index addr = emitPreserveStructAccess(*this, addr, field); + } - // If this is a reference field, load the reference right now. - if (FieldType->isReferenceType()) { - LValue RefLVal = MakeAddrLValue(addr, FieldType, FieldBaseInfo, - FieldTBAAInfo); - if (RecordCVR & Qualifiers::Volatile) - RefLVal.getQuals().addVolatile(); - addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo); - - // Qualifiers on the struct don't apply to the referencee. - RecordCVR = 0; - FieldType = FieldType->getPointeeType(); - } + // If this is a reference field, load the reference right now. + if (FieldType->isReferenceType()) { + LValue RefLVal = + MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo); + if (RecordCVR & Qualifiers::Volatile) + RefLVal.getQuals().addVolatile(); + addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo); + + // Qualifiers on the struct don't apply to the referencee. + RecordCVR = 0; + FieldType = FieldType->getPointeeType(); } // Make sure that the address is pointing to the right type. This is critical @@ -4124,7 +4216,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, if (!FieldType->isReferenceType()) return EmitLValueForField(Base, Field); - Address V = emitAddrOfFieldStorage(*this, Base.getAddress(), Field); + Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field); // Make sure that the address is pointing to the right type. llvm::Type *llvmType = ConvertTypeForMem(FieldType); @@ -4242,10 +4334,10 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { EmitBlock(contBlock); if (lhs && rhs) { - llvm::PHINode *phi = Builder.CreatePHI(lhs->getPointer()->getType(), - 2, "cond-lvalue"); - phi->addIncoming(lhs->getPointer(), lhsBlock); - phi->addIncoming(rhs->getPointer(), rhsBlock); + llvm::PHINode *phi = + Builder.CreatePHI(lhs->getPointer(*this)->getType(), 2, "cond-lvalue"); + phi->addIncoming(lhs->getPointer(*this), lhsBlock); + phi->addIncoming(rhs->getPointer(*this), rhsBlock); Address result(phi, std::min(lhs->getAlignment(), rhs->getAlignment())); AlignmentSource alignSource = std::max(lhs->getBaseInfo().getAlignmentSource(), @@ -4328,7 +4420,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_Dynamic: { LValue LV = EmitLValue(E->getSubExpr()); - Address V = LV.getAddress(); + Address V = LV.getAddress(*this); const auto *DCE = cast<CXXDynamicCastExpr>(E); return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); } @@ -4343,12 +4435,12 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { - const RecordType *DerivedClassTy = - E->getSubExpr()->getType()->getAs<RecordType>(); + const auto *DerivedClassTy = + E->getSubExpr()->getType()->castAs<RecordType>(); auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl()); LValue LV = EmitLValue(E->getSubExpr()); - Address This = LV.getAddress(); + Address This = LV.getAddress(*this); // Perform the derived-to-base conversion Address Base = GetAddressOfBaseClass( @@ -4364,16 +4456,15 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_ToUnion: return EmitAggExprToLValue(E); case CK_BaseToDerived: { - const RecordType *DerivedClassTy = E->getType()->getAs<RecordType>(); + const auto *DerivedClassTy = E->getType()->castAs<RecordType>(); auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl()); LValue LV = EmitLValue(E->getSubExpr()); // Perform the base-to-derived conversion - Address Derived = - GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl, - E->path_begin(), E->path_end(), - /*NullCheckValue=*/false); + Address Derived = GetAddressOfDerivedClass( + LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(), + /*NullCheckValue=*/false); // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is // performed and the object is not of the derived type. @@ -4395,7 +4486,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { CGM.EmitExplicitCastExprType(CE, this); LValue LV = EmitLValue(E->getSubExpr()); - Address V = Builder.CreateBitCast(LV.getAddress(), + Address V = Builder.CreateBitCast(LV.getAddress(*this), ConvertType(CE->getTypeAsWritten())); if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) @@ -4410,14 +4501,15 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); QualType DestTy = getContext().getPointerType(E->getType()); llvm::Value *V = getTargetHooks().performAddrSpaceCast( - *this, LV.getPointer(), E->getSubExpr()->getType().getAddressSpace(), + *this, LV.getPointer(*this), + E->getSubExpr()->getType().getAddressSpace(), E->getType().getAddressSpace(), ConvertType(DestTy)); - return MakeAddrLValue(Address(V, LV.getAddress().getAlignment()), + return MakeAddrLValue(Address(V, LV.getAddress(*this).getAlignment()), E->getType(), LV.getBaseInfo(), LV.getTBAAInfo()); } case CK_ObjCObjectLValueCast: { LValue LV = EmitLValue(E->getSubExpr()); - Address V = Builder.CreateElementBitCast(LV.getAddress(), + Address V = Builder.CreateElementBitCast(LV.getAddress(*this), ConvertType(E->getType())); return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(), CGM.getTBAAInfoForSubobject(LV, E->getType())); @@ -4471,13 +4563,17 @@ RValue CodeGenFunction::EmitRValueForField(LValue LV, case TEK_Complex: return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc)); case TEK_Aggregate: - return FieldLV.asAggregateRValue(); + return FieldLV.asAggregateRValue(*this); case TEK_Scalar: // This routine is used to load fields one-by-one to perform a copy, so // don't load reference fields. if (FD->getType()->isReferenceType()) - return RValue::get(FieldLV.getPointer()); - return EmitLoadOfLValue(FieldLV, Loc); + return RValue::get(FieldLV.getPointer(*this)); + // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a + // primitive load. + if (FieldLV.isBitField()) + return EmitLoadOfLValue(FieldLV, Loc); + return RValue::get(EmitLoadOfScalar(FieldLV, Loc)); } llvm_unreachable("bad evaluation kind"); } @@ -4525,8 +4621,15 @@ RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E, } static CGCallee EmitDirectCallee(CodeGenFunction &CGF, const FunctionDecl *FD) { + if (auto builtinID = FD->getBuiltinID()) { - return CGCallee::forBuiltin(builtinID, FD); + // Replaceable builtin provide their own implementation of a builtin. Unless + // we are in the builtin implementation itself, don't call the actual + // builtin. If we are in the builtin implementation, avoid trivial infinite + // recursion. + if (!FD->isInlineBuiltinDeclaration() || + CGF.CurFn->getName() == FD->getName()) + return CGCallee::forBuiltin(builtinID, FD); } llvm::Constant *calleePtr = EmitFunctionDeclPointer(CGF.CGM, FD); @@ -4571,7 +4674,7 @@ CGCallee CodeGenFunction::EmitCallee(const Expr *E) { functionType = ptrType->getPointeeType(); } else { functionType = E->getType(); - calleePtr = EmitLValue(E).getPointer(); + calleePtr = EmitLValue(E).getPointer(*this); } assert(functionType->isFunctionType()); @@ -4623,6 +4726,9 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) { if (RV.isScalar()) EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc()); EmitStoreThroughLValue(RV, LV); + if (getLangOpts().OpenMP) + CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this, + E->getLHS()); return LV; } @@ -4731,7 +4837,7 @@ LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) { BaseQuals = ObjectTy.getQualifiers(); } else { LValue BaseLV = EmitLValue(BaseExpr); - BaseValue = BaseLV.getPointer(); + BaseValue = BaseLV.getPointer(*this); ObjectTy = BaseExpr->getType(); BaseQuals = ObjectTy.getQualifiers(); } @@ -4941,13 +5047,11 @@ EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) { if (E->getOpcode() == BO_PtrMemI) { BaseAddr = EmitPointerWithAlignment(E->getLHS()); } else { - BaseAddr = EmitLValue(E->getLHS()).getAddress(); + BaseAddr = EmitLValue(E->getLHS()).getAddress(*this); } llvm::Value *OffsetV = EmitScalarExpr(E->getRHS()); - - const MemberPointerType *MPT - = E->getRHS()->getType()->getAs<MemberPointerType>(); + const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>(); LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; @@ -4968,7 +5072,7 @@ RValue CodeGenFunction::convertTempToRValue(Address addr, case TEK_Complex: return RValue::getComplex(EmitLoadOfComplex(lvalue, loc)); case TEK_Aggregate: - return lvalue.asAggregateRValue(); + return lvalue.asAggregateRValue(*this); case TEK_Scalar: return RValue::get(EmitLoadOfScalar(lvalue, loc)); } |