summaryrefslogtreecommitdiff
path: root/lib/CodeGen/CGExprAgg.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/CGExprAgg.cpp')
-rw-r--r--lib/CodeGen/CGExprAgg.cpp454
1 files changed, 362 insertions, 92 deletions
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 0f05cab66d7ee..2917404783298 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -12,8 +12,10 @@
//===----------------------------------------------------------------------===//
#include "CodeGenFunction.h"
+#include "CGCXXABI.h"
#include "CGObjCRuntime.h"
#include "CodeGenModule.h"
+#include "ConstantEmitter.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/DeclTemplate.h"
@@ -22,6 +24,7 @@
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicInst.h"
using namespace clang;
using namespace CodeGen;
@@ -36,23 +39,6 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
AggValueSlot Dest;
bool IsResultUnused;
- /// We want to use 'dest' as the return slot except under two
- /// conditions:
- /// - The destination slot requires garbage collection, so we
- /// need to use the GC API.
- /// - The destination slot is potentially aliased.
- bool shouldUseDestForReturnSlot() const {
- return !(Dest.requiresGCollection() || Dest.isPotentiallyAliased());
- }
-
- ReturnValueSlot getReturnValueSlot() const {
- if (!shouldUseDestForReturnSlot())
- return ReturnValueSlot();
-
- return ReturnValueSlot(Dest.getAddress(), Dest.isVolatile(),
- IsResultUnused);
- }
-
AggValueSlot EnsureSlot(QualType T) {
if (!Dest.isIgnored()) return Dest;
return CGF.CreateAggTemp(T, "agg.tmp.ensured");
@@ -62,6 +48,15 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured");
}
+ // Calls `Fn` with a valid return value slot, potentially creating a temporary
+ // to do so. If a temporary is created, an appropriate copy into `Dest` will
+ // be emitted, as will lifetime markers.
+ //
+ // The given function should take a ReturnValueSlot, and return an RValue that
+ // points to said slot.
+ void withReturnValueSlot(const Expr *E,
+ llvm::function_ref<RValue(ReturnValueSlot)> Fn);
+
public:
AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
: CGF(cgf), Builder(CGF.Builder), Dest(Dest),
@@ -76,8 +71,15 @@ public:
/// then loads the result into DestPtr.
void EmitAggLoadOfLValue(const Expr *E);
+ enum ExprValueKind {
+ EVK_RValue,
+ EVK_NonRValue
+ };
+
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
- void EmitFinalDestCopy(QualType type, const LValue &src);
+ /// SrcIsRValue is true if source comes from an RValue.
+ void EmitFinalDestCopy(QualType type, const LValue &src,
+ ExprValueKind SrcValueKind = EVK_NonRValue);
void EmitFinalDestCopy(QualType type, RValue src);
void EmitCopy(QualType type, const AggValueSlot &dest,
const AggValueSlot &src);
@@ -85,7 +87,7 @@ public:
void EmitMoveFromReturnSlot(const Expr *E, RValue Src);
void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
- QualType elementType, InitListExpr *E);
+ QualType ArrayQTy, InitListExpr *E);
AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
@@ -144,6 +146,7 @@ public:
void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
void VisitBinAssign(const BinaryOperator *E);
void VisitBinComma(const BinaryOperator *E);
+ void VisitBinCmp(const BinaryOperator *E);
void VisitObjCMessageExpr(ObjCMessageExpr *E);
void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
@@ -217,7 +220,7 @@ void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
EmitFinalDestCopy(E->getType(), LV);
}
-/// \brief True if the given aggregate type requires special GC API calls.
+/// True if the given aggregate type requires special GC API calls.
bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
// Only record types have members that might require garbage collection.
const RecordType *RecordTy = T->getAs<RecordType>();
@@ -234,38 +237,78 @@ bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
return Record->hasObjectMember();
}
-/// \brief Perform the final move to DestPtr if for some reason
-/// getReturnValueSlot() didn't use it directly.
-///
-/// The idea is that you do something like this:
-/// RValue Result = EmitSomething(..., getReturnValueSlot());
-/// EmitMoveFromReturnSlot(E, Result);
-///
-/// If nothing interferes, this will cause the result to be emitted
-/// directly into the return value slot. Otherwise, a final move
-/// will be performed.
-void AggExprEmitter::EmitMoveFromReturnSlot(const Expr *E, RValue src) {
- if (shouldUseDestForReturnSlot()) {
- // Logically, Dest.getAddr() should equal Src.getAggregateAddr().
- // The possibility of undef rvalues complicates that a lot,
- // though, so we can't really assert.
- return;
+void AggExprEmitter::withReturnValueSlot(
+ const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
+ QualType RetTy = E->getType();
+ bool RequiresDestruction =
+ Dest.isIgnored() &&
+ RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
+
+ // If it makes no observable difference, save a memcpy + temporary.
+ //
+ // We need to always provide our own temporary if destruction is required.
+ // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
+ // its lifetime before we have the chance to emit a proper destructor call.
+ bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
+ (RequiresDestruction && !Dest.getAddress().isValid());
+
+ Address RetAddr = Address::invalid();
+ Address RetAllocaAddr = Address::invalid();
+
+ EHScopeStack::stable_iterator LifetimeEndBlock;
+ llvm::Value *LifetimeSizePtr = nullptr;
+ llvm::IntrinsicInst *LifetimeStartInst = nullptr;
+ if (!UseTemp) {
+ RetAddr = Dest.getAddress();
+ } else {
+ RetAddr = CGF.CreateMemTemp(RetTy, "tmp", &RetAllocaAddr);
+ uint64_t Size =
+ CGF.CGM.getDataLayout().getTypeAllocSize(CGF.ConvertTypeForMem(RetTy));
+ LifetimeSizePtr = CGF.EmitLifetimeStart(Size, RetAllocaAddr.getPointer());
+ if (LifetimeSizePtr) {
+ LifetimeStartInst =
+ cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint()));
+ assert(LifetimeStartInst->getIntrinsicID() ==
+ llvm::Intrinsic::lifetime_start &&
+ "Last insertion wasn't a lifetime.start?");
+
+ CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
+ NormalEHLifetimeMarker, RetAllocaAddr, LifetimeSizePtr);
+ LifetimeEndBlock = CGF.EHStack.stable_begin();
+ }
}
- // Otherwise, copy from there to the destination.
- assert(Dest.getPointer() != src.getAggregatePointer());
- EmitFinalDestCopy(E->getType(), src);
+ RValue Src =
+ EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused));
+
+ if (RequiresDestruction)
+ CGF.pushDestroy(RetTy.isDestructedType(), Src.getAggregateAddress(), RetTy);
+
+ if (!UseTemp)
+ return;
+
+ assert(Dest.getPointer() != Src.getAggregatePointer());
+ EmitFinalDestCopy(E->getType(), Src);
+
+ if (!RequiresDestruction && LifetimeStartInst) {
+ // If there's no dtor to run, the copy was the last use of our temporary.
+ // Since we're not guaranteed to be in an ExprWithCleanups, clean up
+ // eagerly.
+ CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst);
+ CGF.EmitLifetimeEnd(LifetimeSizePtr, RetAllocaAddr.getPointer());
+ }
}
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
assert(src.isAggregate() && "value must be aggregate value!");
LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type);
- EmitFinalDestCopy(type, srcLV);
+ EmitFinalDestCopy(type, srcLV, EVK_RValue);
}
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
-void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src) {
+void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src,
+ ExprValueKind SrcValueKind) {
// If Dest is ignored, then we're evaluating an aggregate expression
// in a context that doesn't care about the result. Note that loads
// from volatile l-values force the existence of a non-ignored
@@ -273,9 +316,32 @@ void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src) {
if (Dest.isIgnored())
return;
+ // Copy non-trivial C structs here.
+ LValue DstLV = CGF.MakeAddrLValue(
+ Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type);
+
+ if (SrcValueKind == EVK_RValue) {
+ if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
+ if (Dest.isPotentiallyAliased())
+ CGF.callCStructMoveAssignmentOperator(DstLV, src);
+ else
+ CGF.callCStructMoveConstructor(DstLV, src);
+ return;
+ }
+ } else {
+ if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
+ if (Dest.isPotentiallyAliased())
+ CGF.callCStructCopyAssignmentOperator(DstLV, src);
+ else
+ CGF.callCStructCopyConstructor(DstLV, src);
+ return;
+ }
+ }
+
AggValueSlot srcAgg =
AggValueSlot::forLValue(src, AggValueSlot::IsDestructed,
- needsGC(type), AggValueSlot::IsAliased);
+ needsGC(type), AggValueSlot::IsAliased,
+ AggValueSlot::MayOverlap);
EmitCopy(type, Dest, srcAgg);
}
@@ -286,7 +352,7 @@ void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src) {
void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
const AggValueSlot &src) {
if (dest.requiresGCollection()) {
- CharUnits sz = CGF.getContext().getTypeSizeInChars(type);
+ CharUnits sz = dest.getPreferredSize(CGF.getContext(), type);
llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity());
CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
dest.getAddress(),
@@ -298,11 +364,13 @@ void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
// If the result of the assignment is used, copy the LHS there also.
// It's volatile if either side is. Use the minimum alignment of
// the two sides.
- CGF.EmitAggregateCopy(dest.getAddress(), src.getAddress(), type,
+ LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type);
+ LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type);
+ CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(),
dest.isVolatile() || src.isVolatile());
}
-/// \brief Emit the initializer for a std::initializer_list initialized with a
+/// Emit the initializer for a std::initializer_list initialized with a
/// real initializer list.
void
AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
@@ -367,7 +435,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
}
}
-/// \brief Determine if E is a trivial array filler, that is, one that is
+/// Determine if E is a trivial array filler, that is, one that is
/// equivalent to zero-initialization.
static bool isTrivialFiller(Expr *E) {
if (!E)
@@ -390,14 +458,17 @@ static bool isTrivialFiller(Expr *E) {
return false;
}
-/// \brief Emit initialization of an array from an initializer list.
+/// Emit initialization of an array from an initializer list.
void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
- QualType elementType, InitListExpr *E) {
+ QualType ArrayQTy, InitListExpr *E) {
uint64_t NumInitElements = E->getNumInits();
uint64_t NumArrayElements = AType->getNumElements();
assert(NumInitElements <= NumArrayElements);
+ QualType elementType =
+ CGF.getContext().getAsArrayType(ArrayQTy)->getElementType();
+
// DestPtr is an array*. Construct an elementType* by drilling
// down a level.
llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
@@ -409,6 +480,29 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
CharUnits elementAlign =
DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
+ // Consider initializing the array by copying from a global. For this to be
+ // more efficient than per-element initialization, the size of the elements
+ // with explicit initializers should be large enough.
+ if (NumInitElements * elementSize.getQuantity() > 16 &&
+ elementType.isTriviallyCopyableType(CGF.getContext())) {
+ CodeGen::CodeGenModule &CGM = CGF.CGM;
+ ConstantEmitter Emitter(CGM);
+ LangAS AS = ArrayQTy.getAddressSpace();
+ if (llvm::Constant *C = Emitter.tryEmitForInitializer(E, AS, ArrayQTy)) {
+ auto GV = new llvm::GlobalVariable(
+ CGM.getModule(), C->getType(),
+ CGM.isTypeConstant(ArrayQTy, /* ExcludeCtorDtor= */ true),
+ llvm::GlobalValue::PrivateLinkage, C, "constinit",
+ /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
+ CGM.getContext().getTargetAddressSpace(AS));
+ Emitter.finalize(GV);
+ CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy);
+ GV->setAlignment(Align.getQuantity());
+ EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GV, ArrayQTy, Align));
+ return;
+ }
+ }
+
// Exception safety requires us to destroy all the
// already-constructed members if an initializer throws.
// For that, we'll need an EH cleanup.
@@ -540,7 +634,11 @@ void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){
}
void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
- EmitFinalDestCopy(e->getType(), CGF.getOpaqueLValueMapping(e));
+ // If this is a unique OVE, just visit its source expression.
+ if (e->isUnique())
+ Visit(e->getSourceExpr());
+ else
+ EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e));
}
void
@@ -586,12 +684,12 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E));
else
CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");
-
+
if (!Dest.isIgnored())
CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");
break;
}
-
+
case CK_ToUnion: {
// Evaluate even if the destination is ignored.
if (Dest.isIgnored()) {
@@ -651,7 +749,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
if (isToAtomic) {
AggValueSlot valueDest = Dest;
if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) {
- // Zero-initialize. (Strictly speaking, we only need to intialize
+ // Zero-initialize. (Strictly speaking, we only need to initialize
// the padding at the end, but this is simpler.)
if (!Dest.isZeroed())
CGF.EmitNullInitialization(Dest.getAddress(), atomicType);
@@ -665,6 +763,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
valueDest.isExternallyDestructed(),
valueDest.requiresGCollection(),
valueDest.isPotentiallyAliased(),
+ AggValueSlot::DoesNotOverlap,
AggValueSlot::IsZeroed);
}
@@ -762,13 +861,15 @@ void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
return;
}
- RValue RV = CGF.EmitCallExpr(E, getReturnValueSlot());
- EmitMoveFromReturnSlot(E, RV);
+ withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
+ return CGF.EmitCallExpr(E, Slot);
+ });
}
void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
- RValue RV = CGF.EmitObjCMessageExpr(E, getReturnValueSlot());
- EmitMoveFromReturnSlot(E, RV);
+ withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
+ return CGF.EmitObjCMessageExpr(E, Slot);
+ });
}
void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
@@ -781,6 +882,150 @@ void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest);
}
+enum CompareKind {
+ CK_Less,
+ CK_Greater,
+ CK_Equal,
+};
+
+static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
+ const BinaryOperator *E, llvm::Value *LHS,
+ llvm::Value *RHS, CompareKind Kind,
+ const char *NameSuffix = "") {
+ QualType ArgTy = E->getLHS()->getType();
+ if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
+ ArgTy = CT->getElementType();
+
+ if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
+ assert(Kind == CK_Equal &&
+ "member pointers may only be compared for equality");
+ return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
+ CGF, LHS, RHS, MPT, /*IsInequality*/ false);
+ }
+
+ // Compute the comparison instructions for the specified comparison kind.
+ struct CmpInstInfo {
+ const char *Name;
+ llvm::CmpInst::Predicate FCmp;
+ llvm::CmpInst::Predicate SCmp;
+ llvm::CmpInst::Predicate UCmp;
+ };
+ CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
+ using FI = llvm::FCmpInst;
+ using II = llvm::ICmpInst;
+ switch (Kind) {
+ case CK_Less:
+ return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT};
+ case CK_Greater:
+ return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT};
+ case CK_Equal:
+ return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ};
+ }
+ llvm_unreachable("Unrecognised CompareKind enum");
+ }();
+
+ if (ArgTy->hasFloatingRepresentation())
+ return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS,
+ llvm::Twine(InstInfo.Name) + NameSuffix);
+ if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
+ auto Inst =
+ ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
+ return Builder.CreateICmp(Inst, LHS, RHS,
+ llvm::Twine(InstInfo.Name) + NameSuffix);
+ }
+
+ llvm_unreachable("unsupported aggregate binary expression should have "
+ "already been handled");
+}
+
+void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
+ using llvm::BasicBlock;
+ using llvm::PHINode;
+ using llvm::Value;
+ assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
+ E->getRHS()->getType()));
+ const ComparisonCategoryInfo &CmpInfo =
+ CGF.getContext().CompCategories.getInfoForType(E->getType());
+ assert(CmpInfo.Record->isTriviallyCopyable() &&
+ "cannot copy non-trivially copyable aggregate");
+
+ QualType ArgTy = E->getLHS()->getType();
+
+ // TODO: Handle comparing these types.
+ if (ArgTy->isVectorType())
+ return CGF.ErrorUnsupported(
+ E, "aggregate three-way comparison with vector arguments");
+ if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
+ !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
+ !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
+ return CGF.ErrorUnsupported(E, "aggregate three-way comparison");
+ }
+ bool IsComplex = ArgTy->isAnyComplexType();
+
+ // Evaluate the operands to the expression and extract their values.
+ auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
+ RValue RV = CGF.EmitAnyExpr(E);
+ if (RV.isScalar())
+ return {RV.getScalarVal(), nullptr};
+ if (RV.isAggregate())
+ return {RV.getAggregatePointer(), nullptr};
+ assert(RV.isComplex());
+ return RV.getComplexVal();
+ };
+ auto LHSValues = EmitOperand(E->getLHS()),
+ RHSValues = EmitOperand(E->getRHS());
+
+ auto EmitCmp = [&](CompareKind K) {
+ Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first,
+ K, IsComplex ? ".r" : "");
+ if (!IsComplex)
+ return Cmp;
+ assert(K == CompareKind::CK_Equal);
+ Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second,
+ RHSValues.second, K, ".i");
+ return Builder.CreateAnd(Cmp, CmpImag, "and.eq");
+ };
+ auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
+ return Builder.getInt(VInfo->getIntValue());
+ };
+
+ Value *Select;
+ if (ArgTy->isNullPtrType()) {
+ Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
+ } else if (CmpInfo.isEquality()) {
+ Select = Builder.CreateSelect(
+ EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()),
+ EmitCmpRes(CmpInfo.getNonequalOrNonequiv()), "sel.eq");
+ } else if (!CmpInfo.isPartial()) {
+ Value *SelectOne =
+ Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()),
+ EmitCmpRes(CmpInfo.getGreater()), "sel.lt");
+ Select = Builder.CreateSelect(EmitCmp(CK_Equal),
+ EmitCmpRes(CmpInfo.getEqualOrEquiv()),
+ SelectOne, "sel.eq");
+ } else {
+ Value *SelectEq = Builder.CreateSelect(
+ EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()),
+ EmitCmpRes(CmpInfo.getUnordered()), "sel.eq");
+ Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater),
+ EmitCmpRes(CmpInfo.getGreater()),
+ SelectEq, "sel.gt");
+ Select = Builder.CreateSelect(
+ EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt");
+ }
+ // Create the return value in the destination slot.
+ EnsureDest(E->getType());
+ LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
+
+ // Emit the address of the first (and only) field in the comparison category
+ // type, and initialize it from the constant integer value selected above.
+ LValue FieldLV = CGF.EmitLValueForFieldInitialization(
+ DestLV, *CmpInfo.Record->field_begin());
+ CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true);
+
+ // All done! The result is in the Dest slot.
+}
+
void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
VisitPointerToDataMemberBinaryOperator(E);
@@ -890,7 +1135,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
EmitCopy(E->getLHS()->getType(),
AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,
needsGC(E->getLHS()->getType()),
- AggValueSlot::IsAliased),
+ AggValueSlot::IsAliased,
+ AggValueSlot::MayOverlap),
Dest);
return;
}
@@ -911,7 +1157,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
AggValueSlot LHSSlot =
AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,
needsGC(E->getLHS()->getType()),
- AggValueSlot::IsAliased);
+ AggValueSlot::IsAliased,
+ AggValueSlot::MayOverlap);
// A non-volatile aggregate destination might have volatile member.
if (!LHSSlot.isVolatile() &&
CGF.hasVolatileMember(E->getLHS()->getType()))
@@ -1089,6 +1336,7 @@ AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased,
+ AggValueSlot::MayOverlap,
Dest.isZeroed()));
return;
case TEK_Scalar:
@@ -1156,11 +1404,8 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
// Handle initialization of an array.
if (E->getType()->isArrayType()) {
- QualType elementType =
- CGF.getContext().getAsArrayType(E->getType())->getElementType();
-
auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType());
- EmitArrayInit(Dest.getAddress(), AType, elementType, E);
+ EmitArrayInit(Dest.getAddress(), AType, E->getType(), E);
return;
}
@@ -1190,11 +1435,12 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
Dest.getAddress(), CXXRD, BaseRD,
/*isBaseVirtual*/ false);
- AggValueSlot AggSlot =
- AggValueSlot::forAddr(V, Qualifiers(),
- AggValueSlot::IsDestructed,
- AggValueSlot::DoesNotNeedGCBarriers,
- AggValueSlot::IsNotAliased);
+ AggValueSlot AggSlot = AggValueSlot::forAddr(
+ V, Qualifiers(),
+ AggValueSlot::IsDestructed,
+ AggValueSlot::DoesNotNeedGCBarriers,
+ AggValueSlot::IsNotAliased,
+ CGF.overlapForBaseInit(CXXRD, BaseRD, Base.isVirtual()));
CGF.EmitAggExpr(E->getInit(curInitIndex++), AggSlot);
if (QualType::DestructionKind dtorKind =
@@ -1375,7 +1621,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
// If the subexpression is an ArrayInitLoopExpr, share its cleanup.
auto elementSlot = AggValueSlot::forLValue(
elementLV, AggValueSlot::IsDestructed,
- AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased);
+ AggValueSlot::DoesNotNeedGCBarriers,
+ AggValueSlot::IsNotAliased,
+ AggValueSlot::DoesNotOverlap);
AggExprEmitter(CGF, elementSlot, false)
.VisitArrayInitLoopExpr(InnerLoop, outerBegin);
} else
@@ -1425,6 +1673,8 @@ static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
// If this is an initlist expr, sum up the size of sizes of the (present)
// elements. If this is something weird, assume the whole thing is non-zero.
const InitListExpr *ILE = dyn_cast<InitListExpr>(E);
+ while (ILE && ILE->isTransparent())
+ ILE = dyn_cast<InitListExpr>(ILE->getInit(0));
if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType()))
return CGF.getContext().getTypeSizeInChars(E->getType());
@@ -1491,7 +1741,7 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
}
// If the type is 16-bytes or smaller, prefer individual stores over memset.
- CharUnits Size = CGF.getContext().getTypeSizeInChars(E->getType());
+ CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType());
if (Size <= CharUnits::fromQuantity(16))
return;
@@ -1537,16 +1787,42 @@ LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
LValue LV = MakeAddrLValue(Temp, E->getType());
EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
- AggValueSlot::IsNotAliased));
+ AggValueSlot::IsNotAliased,
+ AggValueSlot::DoesNotOverlap));
return LV;
}
-void CodeGenFunction::EmitAggregateCopy(Address DestPtr,
- Address SrcPtr, QualType Ty,
- bool isVolatile,
- bool isAssignment) {
+AggValueSlot::Overlap_t CodeGenFunction::overlapForBaseInit(
+ const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
+ // Virtual bases are initialized first, in address order, so there's never
+ // any overlap during their initialization.
+ //
+ // FIXME: Under P0840, this is no longer true: the tail padding of a vbase
+ // of a field could be reused by a vbase of a containing class.
+ if (IsVirtual)
+ return AggValueSlot::DoesNotOverlap;
+
+ // If the base class is laid out entirely within the nvsize of the derived
+ // class, its tail padding cannot yet be initialized, so we can issue
+ // stores at the full width of the base class.
+ const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
+ if (Layout.getBaseClassOffset(BaseRD) +
+ getContext().getASTRecordLayout(BaseRD).getSize() <=
+ Layout.getNonVirtualSize())
+ return AggValueSlot::DoesNotOverlap;
+
+ // The tail padding may contain values we need to preserve.
+ return AggValueSlot::MayOverlap;
+}
+
+void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
+ AggValueSlot::Overlap_t MayOverlap,
+ bool isVolatile) {
assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
+ Address DestPtr = Dest.getAddress();
+ Address SrcPtr = Src.getAddress();
+
if (getLangOpts().CPlusPlus) {
if (const RecordType *RT = Ty->getAs<RecordType>()) {
CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl());
@@ -1562,7 +1838,7 @@ void CodeGenFunction::EmitAggregateCopy(Address DestPtr,
return;
}
}
-
+
// Aggregate assignment turns into llvm.memcpy. This is almost valid per
// C99 6.5.16.1p3, which states "If the value being stored in an object is
// read from another object that overlaps in anyway the storage of the first
@@ -1574,12 +1850,11 @@ void CodeGenFunction::EmitAggregateCopy(Address DestPtr,
// implementation handles this case safely. If there is a libc that does not
// safely handle this, we can add a target hook.
- // Get data size info for this aggregate. If this is an assignment,
- // don't copy the tail padding, because we might be assigning into a
- // base subobject where the tail padding is claimed. Otherwise,
- // copying it is fine.
+ // Get data size info for this aggregate. Don't copy the tail padding if this
+ // might be a potentially-overlapping subobject, since the tail padding might
+ // be occupied by a different object. Otherwise, copying it is fine.
std::pair<CharUnits, CharUnits> TypeInfo;
- if (isAssignment)
+ if (MayOverlap)
TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);
else
TypeInfo = getContext().getTypeInfoInChars(Ty);
@@ -1591,22 +1866,11 @@ void CodeGenFunction::EmitAggregateCopy(Address DestPtr,
getContext().getAsArrayType(Ty))) {
QualType BaseEltTy;
SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);
- TypeInfo = getContext().getTypeInfoDataSizeInChars(BaseEltTy);
- std::pair<CharUnits, CharUnits> LastElementTypeInfo;
- if (!isAssignment)
- LastElementTypeInfo = getContext().getTypeInfoInChars(BaseEltTy);
+ TypeInfo = getContext().getTypeInfoInChars(BaseEltTy);
assert(!TypeInfo.first.isZero());
SizeVal = Builder.CreateNUWMul(
SizeVal,
llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()));
- if (!isAssignment) {
- SizeVal = Builder.CreateNUWSub(
- SizeVal,
- llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()));
- SizeVal = Builder.CreateNUWAdd(
- SizeVal, llvm::ConstantInt::get(
- SizeTy, LastElementTypeInfo.first.getQuantity()));
- }
}
}
if (!SizeVal) {
@@ -1657,4 +1921,10 @@ void CodeGenFunction::EmitAggregateCopy(Address DestPtr,
// the optimizer wishes to expand it in to scalar memory operations.
if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty))
Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag);
+
+ if (CGM.getCodeGenOpts().NewStructPathTBAA) {
+ TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
+ Dest.getTBAAInfo(), Src.getTBAAInfo());
+ CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
+ }
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-29 23:06:14 +0000