summaryrefslogtreecommitdiff
path: root/clang/lib/CodeGen/CGCall.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'clang/lib/CodeGen/CGCall.cpp')
-rw-r--r--clang/lib/CodeGen/CGCall.cpp853
1 files changed, 656 insertions, 197 deletions
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index e4803fde230f0..e8235c775d8f5 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -16,6 +16,7 @@
#include "CGBlocks.h"
#include "CGCXXABI.h"
#include "CGCleanup.h"
+#include "CGRecordLayout.h"
#include "CodeGenFunction.h"
#include "CodeGenModule.h"
#include "TargetInfo.h"
@@ -325,7 +326,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
if (PassParams)
appendParameterTypes(*this, argTypes, paramInfos, FTP);
- CGCXXABI::AddedStructorArgs AddedArgs =
+ CGCXXABI::AddedStructorArgCounts AddedArgs =
TheCXXABI.buildStructorSignature(GD, argTypes);
if (!paramInfos.empty()) {
// Note: prefix implies after the first param.
@@ -815,6 +816,7 @@ CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
FI->ASTCallingConvention = info.getCC();
FI->InstanceMethod = instanceMethod;
FI->ChainCall = chainCall;
+ FI->CmseNSCall = info.getCmseNSCall();
FI->NoReturn = info.getNoReturn();
FI->ReturnsRetained = info.getProducesResult();
FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
@@ -1014,8 +1016,8 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF,
}
}
-void CodeGenFunction::ExpandTypeFromArgs(
- QualType Ty, LValue LV, SmallVectorImpl<llvm::Value *>::iterator &AI) {
+void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
+ llvm::Function::arg_iterator &AI) {
assert(LV.isSimple() &&
"Unexpected non-simple lvalue during struct expansion.");
@@ -1044,17 +1046,17 @@ void CodeGenFunction::ExpandTypeFromArgs(
ExpandTypeFromArgs(FD->getType(), SubLV, AI);
}
} else if (isa<ComplexExpansion>(Exp.get())) {
- auto realValue = *AI++;
- auto imagValue = *AI++;
+ auto realValue = &*AI++;
+ auto imagValue = &*AI++;
EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
} else {
// Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
// primitive store.
assert(isa<NoExpansion>(Exp.get()));
if (LV.isBitField())
- EmitStoreThroughLValue(RValue::get(*AI++), LV);
+ EmitStoreThroughLValue(RValue::get(&*AI++), LV);
else
- EmitStoreOfScalar(*AI++, LV);
+ EmitStoreOfScalar(&*AI++, LV);
}
}
@@ -1232,7 +1234,7 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, DstSize, CGF);
- SrcTy = Src.getType()->getElementType();
+ SrcTy = Src.getElementType();
}
uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
@@ -1260,11 +1262,9 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
// Otherwise do coercion through memory. This is stupid, but simple.
Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment());
- Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
- Address SrcCasted = CGF.Builder.CreateElementBitCast(Src,CGF.Int8Ty);
- CGF.Builder.CreateMemCpy(Casted, SrcCasted,
- llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
- false);
+ CGF.Builder.CreateMemCpy(Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
+ Src.getPointer(), Src.getAlignment().getAsAlign(),
+ llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize));
return CGF.Builder.CreateLoad(Tmp);
}
@@ -1272,18 +1272,17 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
// store the elements rather than the aggregate to be more friendly to
// fast-isel.
// FIXME: Do we need to recurse here?
-static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
- Address Dest, bool DestIsVolatile) {
+void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
+ bool DestIsVolatile) {
// Prefer scalar stores to first-class aggregate stores.
- if (llvm::StructType *STy =
- dyn_cast<llvm::StructType>(Val->getType())) {
+ if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i);
- llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
- CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
+ Address EltPtr = Builder.CreateStructGEP(Dest, i);
+ llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
+ Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
}
} else {
- CGF.Builder.CreateStore(Val, Dest, DestIsVolatile);
+ Builder.CreateStore(Val, Dest, DestIsVolatile);
}
}
@@ -1298,7 +1297,7 @@ static void CreateCoercedStore(llvm::Value *Src,
bool DstIsVolatile,
CodeGenFunction &CGF) {
llvm::Type *SrcTy = Src->getType();
- llvm::Type *DstTy = Dst.getType()->getElementType();
+ llvm::Type *DstTy = Dst.getElementType();
if (SrcTy == DstTy) {
CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
return;
@@ -1308,7 +1307,7 @@ static void CreateCoercedStore(llvm::Value *Src,
if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, SrcSize, CGF);
- DstTy = Dst.getType()->getElementType();
+ DstTy = Dst.getElementType();
}
llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
@@ -1334,7 +1333,7 @@ static void CreateCoercedStore(llvm::Value *Src,
// If store is legal, just bitcast the src pointer.
if (SrcSize <= DstSize) {
Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
- BuildAggStore(CGF, Src, Dst, DstIsVolatile);
+ CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
} else {
// Otherwise do coercion through memory. This is stupid, but
// simple.
@@ -1347,11 +1346,9 @@ static void CreateCoercedStore(llvm::Value *Src,
// to that information.
Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
CGF.Builder.CreateStore(Src, Tmp);
- Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
- Address DstCasted = CGF.Builder.CreateElementBitCast(Dst,CGF.Int8Ty);
- CGF.Builder.CreateMemCpy(DstCasted, Casted,
- llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
- false);
+ CGF.Builder.CreateMemCpy(Dst.getPointer(), Dst.getAlignment().getAsAlign(),
+ Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
+ llvm::ConstantInt::get(CGF.IntPtrTy, DstSize));
}
}
@@ -1702,8 +1699,9 @@ static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
}
-void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
- bool AttrOnCallSite,
+void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
+ bool HasOptnone,
+ bool AttrOnCallSite,
llvm::AttrBuilder &FuncAttrs) {
// OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
if (!HasOptnone) {
@@ -1746,13 +1744,20 @@ void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
if (CodeGenOpts.NullPointerIsValid)
- FuncAttrs.addAttribute("null-pointer-is-valid", "true");
- if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::Invalid)
+ FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
+
+ if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
FuncAttrs.addAttribute("denormal-fp-math",
- llvm::denormalModeName(CodeGenOpts.FPDenormalMode));
+ CodeGenOpts.FPDenormalMode.str());
+ if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
+ FuncAttrs.addAttribute(
+ "denormal-fp-math-f32",
+ CodeGenOpts.FP32DenormalMode.str());
+ }
FuncAttrs.addAttribute("no-trapping-math",
- llvm::toStringRef(CodeGenOpts.NoTrappingMath));
+ llvm::toStringRef(LangOpts.getFPExceptionMode() ==
+ LangOptions::FPE_Ignore));
// Strict (compliant) code is the default, so only add this attribute to
// indicate that we are trying to workaround a problem case.
@@ -1762,25 +1767,21 @@ void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
// TODO: Are these all needed?
// unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
FuncAttrs.addAttribute("no-infs-fp-math",
- llvm::toStringRef(CodeGenOpts.NoInfsFPMath));
+ llvm::toStringRef(LangOpts.NoHonorInfs));
FuncAttrs.addAttribute("no-nans-fp-math",
- llvm::toStringRef(CodeGenOpts.NoNaNsFPMath));
+ llvm::toStringRef(LangOpts.NoHonorNaNs));
FuncAttrs.addAttribute("unsafe-fp-math",
- llvm::toStringRef(CodeGenOpts.UnsafeFPMath));
+ llvm::toStringRef(LangOpts.UnsafeFPMath));
FuncAttrs.addAttribute("use-soft-float",
llvm::toStringRef(CodeGenOpts.SoftFloat));
FuncAttrs.addAttribute("stack-protector-buffer-size",
llvm::utostr(CodeGenOpts.SSPBufferSize));
FuncAttrs.addAttribute("no-signed-zeros-fp-math",
- llvm::toStringRef(CodeGenOpts.NoSignedZeros));
+ llvm::toStringRef(LangOpts.NoSignedZero));
FuncAttrs.addAttribute(
"correctly-rounded-divide-sqrt-fp-math",
llvm::toStringRef(CodeGenOpts.CorrectlyRoundedDivSqrt));
- if (getLangOpts().OpenCL)
- FuncAttrs.addAttribute("denorms-are-zero",
- llvm::toStringRef(CodeGenOpts.FlushDenorm));
-
// TODO: Reciprocal estimate codegen options should apply to instructions?
const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
if (!Recips.empty())
@@ -1796,6 +1797,8 @@ void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
FuncAttrs.addAttribute("stackrealign");
if (CodeGenOpts.Backchain)
FuncAttrs.addAttribute("backchain");
+ if (CodeGenOpts.EnableSegmentedStacks)
+ FuncAttrs.addAttribute("split-stack");
if (CodeGenOpts.SpeculativeLoadHardening)
FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
@@ -1813,10 +1816,6 @@ void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) {
// Exceptions aren't supported in CUDA device code.
FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
-
- // Respect -fcuda-flush-denormals-to-zero.
- if (CodeGenOpts.FlushDenorm)
- FuncAttrs.addAttribute("nvptx-f32ftz", "true");
}
for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
@@ -1826,31 +1825,100 @@ void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
}
}
-void CodeGenModule::AddDefaultFnAttrs(llvm::Function &F) {
+void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
llvm::AttrBuilder FuncAttrs;
- ConstructDefaultFnAttrList(F.getName(), F.hasOptNone(),
- /* AttrOnCallSite = */ false, FuncAttrs);
+ getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
+ /* AttrOnCallSite = */ false, FuncAttrs);
+ // TODO: call GetCPUAndFeaturesAttributes?
F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
}
+void CodeGenModule::addDefaultFunctionDefinitionAttributes(
+ llvm::AttrBuilder &attrs) {
+ getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
+ /*for call*/ false, attrs);
+ GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
+}
+
+static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
+ const LangOptions &LangOpts,
+ const NoBuiltinAttr *NBA = nullptr) {
+ auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
+ SmallString<32> AttributeName;
+ AttributeName += "no-builtin-";
+ AttributeName += BuiltinName;
+ FuncAttrs.addAttribute(AttributeName);
+ };
+
+ // First, handle the language options passed through -fno-builtin.
+ if (LangOpts.NoBuiltin) {
+ // -fno-builtin disables them all.
+ FuncAttrs.addAttribute("no-builtins");
+ return;
+ }
+
+ // Then, add attributes for builtins specified through -fno-builtin-<name>.
+ llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
+
+ // Now, let's check the __attribute__((no_builtin("...")) attribute added to
+ // the source.
+ if (!NBA)
+ return;
+
+ // If there is a wildcard in the builtin names specified through the
+ // attribute, disable them all.
+ if (llvm::is_contained(NBA->builtinNames(), "*")) {
+ FuncAttrs.addAttribute("no-builtins");
+ return;
+ }
+
+ // And last, add the rest of the builtin names.
+ llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
+}
+
+/// Construct the IR attribute list of a function or call.
+///
+/// When adding an attribute, please consider where it should be handled:
+///
+/// - getDefaultFunctionAttributes is for attributes that are essentially
+/// part of the global target configuration (but perhaps can be
+/// overridden on a per-function basis). Adding attributes there
+/// will cause them to also be set in frontends that build on Clang's
+/// target-configuration logic, as well as for code defined in library
+/// modules such as CUDA's libdevice.
+///
+/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
+/// and adds declaration-specific, convention-specific, and
+/// frontend-specific logic. The last is of particular importance:
+/// attributes that restrict how the frontend generates code must be
+/// added here rather than getDefaultFunctionAttributes.
+///
void CodeGenModule::ConstructAttributeList(
StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
llvm::AttrBuilder FuncAttrs;
llvm::AttrBuilder RetAttrs;
+ // Collect function IR attributes from the CC lowering.
+ // We'll collect the paramete and result attributes later.
CallingConv = FI.getEffectiveCallingConvention();
if (FI.isNoReturn())
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
+ if (FI.isCmseNSCall())
+ FuncAttrs.addAttribute("cmse_nonsecure_call");
- // If we have information about the function prototype, we can learn
- // attributes from there.
+ // Collect function IR attributes from the callee prototype if we have one.
AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
CalleeInfo.getCalleeFunctionProtoType());
const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
bool HasOptnone = false;
+ // The NoBuiltinAttr attached to the target FunctionDecl.
+ const NoBuiltinAttr *NBA = nullptr;
+
+ // Collect function IR attributes based on declaration-specific
+ // information.
// FIXME: handle sseregparm someday...
if (TargetDecl) {
if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
@@ -1869,6 +1937,13 @@ void CodeGenModule::ConstructAttributeList(
if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
AddAttributesFromFunctionProtoType(
getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
+ if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
+ // A sane operator new returns a non-aliasing pointer.
+ auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
+ if (getCodeGenOpts().AssumeSaneOperatorNew &&
+ (Kind == OO_New || Kind == OO_Array_New))
+ RetAttrs.addAttribute(llvm::Attribute::NoAlias);
+ }
const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
const bool IsVirtualCall = MD && MD->isVirtual();
// Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
@@ -1876,22 +1951,7 @@ void CodeGenModule::ConstructAttributeList(
if (!(AttrOnCallSite && IsVirtualCall)) {
if (Fn->isNoReturn())
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
-
- const auto *NBA = Fn->getAttr<NoBuiltinAttr>();
- bool HasWildcard = NBA && llvm::is_contained(NBA->builtinNames(), "*");
- if (getLangOpts().NoBuiltin || HasWildcard)
- FuncAttrs.addAttribute("no-builtins");
- else {
- auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
- SmallString<32> AttributeName;
- AttributeName += "no-builtin-";
- AttributeName += BuiltinName;
- FuncAttrs.addAttribute(AttributeName);
- };
- llvm::for_each(getLangOpts().NoBuiltinFuncs, AddNoBuiltinAttr);
- if (NBA)
- llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
- }
+ NBA = Fn->getAttr<NoBuiltinAttr>();
}
}
@@ -1924,70 +1984,93 @@ void CodeGenModule::ConstructAttributeList(
FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
NumElemsParam);
}
+
+ if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
+ if (getLangOpts().OpenCLVersion <= 120) {
+ // OpenCL v1.2 Work groups are always uniform
+ FuncAttrs.addAttribute("uniform-work-group-size", "true");
+ } else {
+ // OpenCL v2.0 Work groups may be whether uniform or not.
+ // '-cl-uniform-work-group-size' compile option gets a hint
+ // to the compiler that the global work-size be a multiple of
+ // the work-group size specified to clEnqueueNDRangeKernel
+ // (i.e. work groups are uniform).
+ FuncAttrs.addAttribute("uniform-work-group-size",
+ llvm::toStringRef(CodeGenOpts.UniformWGSize));
+ }
+ }
}
- ConstructDefaultFnAttrList(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
+ // Attach "no-builtins" attributes to:
+ // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
+ // * definitions: "no-builtins" or "no-builtin-<name>" only.
+ // The attributes can come from:
+ // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
+ // * FunctionDecl attributes: __attribute__((no_builtin(...)))
+ addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
+
+ // Collect function IR attributes based on global settiings.
+ getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
- // This must run after constructing the default function attribute list
- // to ensure that the speculative load hardening attribute is removed
- // in the case where the -mspeculative-load-hardening flag was passed.
+ // Override some default IR attributes based on declaration-specific
+ // information.
if (TargetDecl) {
if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
- }
-
- if (CodeGenOpts.EnableSegmentedStacks &&
- !(TargetDecl && TargetDecl->hasAttr<NoSplitStackAttr>()))
- FuncAttrs.addAttribute("split-stack");
-
- // Add NonLazyBind attribute to function declarations when -fno-plt
- // is used.
- if (TargetDecl && CodeGenOpts.NoPLT) {
- if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
- if (!Fn->isDefined() && !AttrOnCallSite) {
- FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
+ if (TargetDecl->hasAttr<NoSplitStackAttr>())
+ FuncAttrs.removeAttribute("split-stack");
+
+ // Add NonLazyBind attribute to function declarations when -fno-plt
+ // is used.
+ // FIXME: what if we just haven't processed the function definition
+ // yet, or if it's an external definition like C99 inline?
+ if (CodeGenOpts.NoPLT) {
+ if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
+ if (!Fn->isDefined() && !AttrOnCallSite) {
+ FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
+ }
}
}
}
- if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>()) {
- if (getLangOpts().OpenCLVersion <= 120) {
- // OpenCL v1.2 Work groups are always uniform
- FuncAttrs.addAttribute("uniform-work-group-size", "true");
- } else {
- // OpenCL v2.0 Work groups may be whether uniform or not.
- // '-cl-uniform-work-group-size' compile option gets a hint
- // to the compiler that the global work-size be a multiple of
- // the work-group size specified to clEnqueueNDRangeKernel
- // (i.e. work groups are uniform).
- FuncAttrs.addAttribute("uniform-work-group-size",
- llvm::toStringRef(CodeGenOpts.UniformWGSize));
- }
- }
-
+ // Collect non-call-site function IR attributes from declaration-specific
+ // information.
if (!AttrOnCallSite) {
- bool DisableTailCalls = false;
+ if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
+ FuncAttrs.addAttribute("cmse_nonsecure_entry");
+
+ // Whether tail calls are enabled.
+ auto shouldDisableTailCalls = [&] {
+ // Should this be honored in getDefaultFunctionAttributes?
+ if (CodeGenOpts.DisableTailCalls)
+ return true;
+
+ if (!TargetDecl)
+ return false;
- if (CodeGenOpts.DisableTailCalls)
- DisableTailCalls = true;
- else if (TargetDecl) {
if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
TargetDecl->hasAttr<AnyX86InterruptAttr>())
- DisableTailCalls = true;
- else if (CodeGenOpts.NoEscapingBlockTailCalls) {
+ return true;
+
+ if (CodeGenOpts.NoEscapingBlockTailCalls) {
if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
if (!BD->doesNotEscape())
- DisableTailCalls = true;
+ return true;
}
- }
+ return false;
+ };
FuncAttrs.addAttribute("disable-tail-calls",
- llvm::toStringRef(DisableTailCalls));
+ llvm::toStringRef(shouldDisableTailCalls()));
+
+ // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
+ // handles these separately to set them based on the global defaults.
GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
}
+ // Collect attributes from arguments and return values.
ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
QualType RetTy = FI.getReturnType();
@@ -2024,11 +2107,16 @@ void CodeGenModule::ConstructAttributeList(
if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
QualType PTy = RefTy->getPointeeType();
if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
- RetAttrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
- .getQuantity());
- else if (getContext().getTargetAddressSpace(PTy) == 0 &&
- !CodeGenOpts.NullPointerIsValid)
+ RetAttrs.addDereferenceableAttr(
+ getMinimumObjectSize(PTy).getQuantity());
+ if (getContext().getTargetAddressSpace(PTy) == 0 &&
+ !CodeGenOpts.NullPointerIsValid)
RetAttrs.addAttribute(llvm::Attribute::NonNull);
+ if (PTy->isObjectType()) {
+ llvm::Align Alignment =
+ getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
+ RetAttrs.addAlignmentAttr(Alignment);
+ }
}
bool hasUsedSRet = false;
@@ -2041,6 +2129,7 @@ void CodeGenModule::ConstructAttributeList(
hasUsedSRet = true;
if (RetAI.getInReg())
SRETAttrs.addAttribute(llvm::Attribute::InReg);
+ SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
}
@@ -2134,11 +2223,16 @@ void CodeGenModule::ConstructAttributeList(
if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
QualType PTy = RefTy->getPointeeType();
if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
- Attrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
- .getQuantity());
- else if (getContext().getTargetAddressSpace(PTy) == 0 &&
- !CodeGenOpts.NullPointerIsValid)
+ Attrs.addDereferenceableAttr(
+ getMinimumObjectSize(PTy).getQuantity());
+ if (getContext().getTargetAddressSpace(PTy) == 0 &&
+ !CodeGenOpts.NullPointerIsValid)
Attrs.addAttribute(llvm::Attribute::NonNull);
+ if (PTy->isObjectType()) {
+ llvm::Align Alignment =
+ getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
+ Attrs.addAlignmentAttr(Alignment);
+ }
}
switch (FI.getExtParameterInfo(ArgNo).getABI()) {
@@ -2161,8 +2255,7 @@ void CodeGenModule::ConstructAttributeList(
if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
auto info = getContext().getTypeInfoInChars(PTy);
Attrs.addDereferenceableAttr(info.first.getQuantity());
- Attrs.addAttribute(llvm::Attribute::getWithAlignment(
- getLLVMContext(), info.second.getAsAlign()));
+ Attrs.addAlignmentAttr(info.second.getAsAlign());
}
break;
}
@@ -2278,19 +2371,13 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
// simplify.
ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
- // Flattened function arguments.
- SmallVector<llvm::Value *, 16> FnArgs;
- FnArgs.reserve(IRFunctionArgs.totalIRArgs());
- for (auto &Arg : Fn->args()) {
- FnArgs.push_back(&Arg);
- }
- assert(FnArgs.size() == IRFunctionArgs.totalIRArgs());
+ assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
// If we're using inalloca, all the memory arguments are GEPs off of the last
// parameter, which is a pointer to the complete memory area.
Address ArgStruct = Address::invalid();
if (IRFunctionArgs.hasInallocaArg()) {
- ArgStruct = Address(FnArgs[IRFunctionArgs.getInallocaArgNo()],
+ ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
FI.getArgStructAlignment());
assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
@@ -2298,7 +2385,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
// Name the struct return parameter.
if (IRFunctionArgs.hasSRetArg()) {
- auto AI = cast<llvm::Argument>(FnArgs[IRFunctionArgs.getSRetArgNo()]);
+ auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
AI->setName("agg.result");
AI->addAttr(llvm::Attribute::NoAlias);
}
@@ -2340,13 +2427,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
auto FieldIndex = ArgI.getInAllocaFieldIndex();
Address V =
Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
+ if (ArgI.getInAllocaIndirect())
+ V = Address(Builder.CreateLoad(V),
+ getContext().getTypeAlignInChars(Ty));
ArgVals.push_back(ParamValue::forIndirect(V));
break;
}
case ABIArgInfo::Indirect: {
assert(NumIRArgs == 1);
- Address ParamAddr = Address(FnArgs[FirstIRArg], ArgI.getIndirectAlign());
+ Address ParamAddr =
+ Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign());
if (!hasScalarEvaluationKind(Ty)) {
// Aggregates and complex variables are accessed by reference. All we
@@ -2361,10 +2452,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
// FIXME: We should have a common utility for generating an aggregate
// copy.
CharUnits Size = getContext().getTypeSizeInChars(Ty);
- auto SizeVal = llvm::ConstantInt::get(IntPtrTy, Size.getQuantity());
- Address Dst = Builder.CreateBitCast(AlignedTemp, Int8PtrTy);
- Address Src = Builder.CreateBitCast(ParamAddr, Int8PtrTy);
- Builder.CreateMemCpy(Dst, Src, SizeVal, false);
+ Builder.CreateMemCpy(
+ AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
+ ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
+ llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
V = AlignedTemp;
}
ArgVals.push_back(ParamValue::forIndirect(V));
@@ -2382,16 +2473,18 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
case ABIArgInfo::Extend:
case ABIArgInfo::Direct: {
-
- // If we have the trivial case, handle it with no muss and fuss.
- if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
- ArgI.getCoerceToType() == ConvertType(Ty) &&
- ArgI.getDirectOffset() == 0) {
+ auto AI = Fn->getArg(FirstIRArg);
+ llvm::Type *LTy = ConvertType(Arg->getType());
+
+ // Prepare parameter attributes. So far, only attributes for pointer
+ // parameters are prepared. See
+ // http://llvm.org/docs/LangRef.html#paramattrs.
+ if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
+ ArgI.getCoerceToType()->isPointerTy()) {
assert(NumIRArgs == 1);
- llvm::Value *V = FnArgs[FirstIRArg];
- auto AI = cast<llvm::Argument>(V);
if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
+ // Set `nonnull` attribute if any.
if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
PVD->getFunctionScopeIndex()) &&
!CGM.getCodeGenOpts().NullPointerIsValid)
@@ -2411,9 +2504,11 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
ArrSize) {
llvm::AttrBuilder Attrs;
Attrs.addDereferenceableAttr(
- getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
+ getContext().getTypeSizeInChars(ETy).getQuantity() *
+ ArrSize);
AI->addAttrs(Attrs);
- } else if (getContext().getTargetAddressSpace(ETy) == 0 &&
+ } else if (getContext().getTargetInfo().getNullPointerValue(
+ ETy.getAddressSpace()) == 0 &&
!CGM.getCodeGenOpts().NullPointerIsValid) {
AI->addAttr(llvm::Attribute::NonNull);
}
@@ -2429,6 +2524,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
AI->addAttr(llvm::Attribute::NonNull);
}
+ // Set `align` attribute if any.
const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
if (!AVAttr)
if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
@@ -2437,21 +2533,33 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
// If alignment-assumption sanitizer is enabled, we do *not* add
// alignment attribute here, but emit normal alignment assumption,
// so the UBSAN check could function.
- llvm::Value *AlignmentValue =
- EmitScalarExpr(AVAttr->getAlignment());
llvm::ConstantInt *AlignmentCI =
- cast<llvm::ConstantInt>(AlignmentValue);
- unsigned Alignment = std::min((unsigned)AlignmentCI->getZExtValue(),
- +llvm::Value::MaximumAlignment);
- AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
+ cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
+ unsigned AlignmentInt =
+ AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
+ if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
+ AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
+ AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(
+ llvm::Align(AlignmentInt)));
+ }
}
}
+ // Set 'noalias' if an argument type has the `restrict` qualifier.
if (Arg->getType().isRestrictQualified())
AI->addAttr(llvm::Attribute::NoAlias);
+ }
+
+ // Prepare the argument value. If we have the trivial case, handle it
+ // with no muss and fuss.
+ if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
+ ArgI.getCoerceToType() == ConvertType(Ty) &&
+ ArgI.getDirectOffset() == 0) {
+ assert(NumIRArgs == 1);
// LLVM expects swifterror parameters to be used in very restricted
// ways. Copy the value into a less-restricted temporary.
+ llvm::Value *V = AI;
if (FI.getExtParameterInfo(ArgNo).getABI()
== ParameterABI::SwiftErrorResult) {
QualType pointeeTy = Ty->getPointeeType();
@@ -2513,7 +2621,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
assert(STy->getNumElements() == NumIRArgs);
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- auto AI = FnArgs[FirstIRArg + i];
+ auto AI = Fn->getArg(FirstIRArg + i);
AI->setName(Arg->getName() + ".coerce" + Twine(i));
Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
Builder.CreateStore(AI, EltPtr);
@@ -2526,7 +2634,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
} else {
// Simple case, just do a coerced store of the argument into the alloca.
assert(NumIRArgs == 1);
- auto AI = FnArgs[FirstIRArg];
+ auto AI = Fn->getArg(FirstIRArg);
AI->setName(Arg->getName() + ".coerce");
CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
}
@@ -2559,7 +2667,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
continue;
auto eltAddr = Builder.CreateStructGEP(alloca, i);
- auto elt = FnArgs[argIndex++];
+ auto elt = Fn->getArg(argIndex++);
Builder.CreateStore(elt, eltAddr);
}
assert(argIndex == FirstIRArg + NumIRArgs);
@@ -2574,11 +2682,11 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
LValue LV = MakeAddrLValue(Alloca, Ty);
ArgVals.push_back(ParamValue::forIndirect(Alloca));
- auto FnArgIter = FnArgs.begin() + FirstIRArg;
+ auto FnArgIter = Fn->arg_begin() + FirstIRArg;
ExpandTypeFromArgs(Ty, LV, FnArgIter);
- assert(FnArgIter == FnArgs.begin() + FirstIRArg + NumIRArgs);
+ assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
- auto AI = FnArgs[FirstIRArg + i];
+ auto AI = Fn->getArg(FirstIRArg + i);
AI->setName(Arg->getName() + "." + Twine(i));
}
break;
@@ -2655,10 +2763,10 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
bool doRetainAutorelease;
- if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints().objc_retain) {
+ if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
doRetainAutorelease = true;
- } else if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints()
- .objc_retainAutoreleasedReturnValue) {
+ } else if (call->getCalledOperand() ==
+ CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
doRetainAutorelease = false;
// If we emitted an assembly marker for this call (and the
@@ -2674,8 +2782,8 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
assert(prev);
}
assert(isa<llvm::CallInst>(prev));
- assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
- CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
+ assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
+ CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
InstsToKill.push_back(prev);
}
} else {
@@ -2718,8 +2826,8 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
// Look for a retain call.
llvm::CallInst *retainCall =
dyn_cast<llvm::CallInst>(result->stripPointerCasts());
- if (!retainCall ||
- retainCall->getCalledValue() != CGF.CGM.getObjCEntrypoints().objc_retain)
+ if (!retainCall || retainCall->getCalledOperand() !=
+ CGF.CGM.getObjCEntrypoints().objc_retain)
return nullptr;
// Look for an ordinary load of 'self'.
@@ -2825,6 +2933,199 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
return store;
}
+// Helper functions for EmitCMSEClearRecord
+
+// Set the bits corresponding to a field having width `BitWidth` and located at
+// offset `BitOffset` (from the least significant bit) within a storage unit of
+// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
+// Use little-endian layout, i.e.`Bits[0]` is the LSB.
+static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
+ int BitWidth, int CharWidth) {
+ assert(CharWidth <= 64);
+ assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
+
+ int Pos = 0;
+ if (BitOffset >= CharWidth) {
+ Pos += BitOffset / CharWidth;
+ BitOffset = BitOffset % CharWidth;
+ }
+
+ const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
+ if (BitOffset + BitWidth >= CharWidth) {
+ Bits[Pos++] |= (Used << BitOffset) & Used;
+ BitWidth -= CharWidth - BitOffset;
+ BitOffset = 0;
+ }
+
+ while (BitWidth >= CharWidth) {
+ Bits[Pos++] = Used;
+ BitWidth -= CharWidth;
+ }
+
+ if (BitWidth > 0)
+ Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
+}
+
+// Set the bits corresponding to a field having width `BitWidth` and located at
+// offset `BitOffset` (from the least significant bit) within a storage unit of
+// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
+// `Bits` corresponds to one target byte. Use target endian layout.
+static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
+ int StorageSize, int BitOffset, int BitWidth,
+ int CharWidth, bool BigEndian) {
+
+ SmallVector<uint64_t, 8> TmpBits(StorageSize);
+ setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
+
+ if (BigEndian)
+ std::reverse(TmpBits.begin(), TmpBits.end());
+
+ for (uint64_t V : TmpBits)
+ Bits[StorageOffset++] |= V;
+}
+
+static void setUsedBits(CodeGenModule &, QualType, int,
+ SmallVectorImpl<uint64_t> &);
+
+// Set the bits in `Bits`, which correspond to the value representations of
+// the actual members of the record type `RTy`. Note that this function does
+// not handle base classes, virtual tables, etc, since they cannot happen in
+// CMSE function arguments or return. The bit mask corresponds to the target
+// memory layout, i.e. it's endian dependent.
+static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
+ SmallVectorImpl<uint64_t> &Bits) {
+ ASTContext &Context = CGM.getContext();
+ int CharWidth = Context.getCharWidth();
+ const RecordDecl *RD = RTy->getDecl()->getDefinition();
+ const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
+ const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
+
+ int Idx = 0;
+ for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
+ const FieldDecl *F = *I;
+
+ if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context) ||
+ F->getType()->isIncompleteArrayType())
+ continue;
+
+ if (F->isBitField()) {
+ const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
+ setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
+ BFI.StorageSize / CharWidth, BFI.Offset,
+ BFI.Size, CharWidth,
+ CGM.getDataLayout().isBigEndian());
+ continue;
+ }
+
+ setUsedBits(CGM, F->getType(),
+ Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
+ }
+}
+
+// Set the bits in `Bits`, which correspond to the value representations of
+// the elements of an array type `ATy`.
+static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
+ int Offset, SmallVectorImpl<uint64_t> &Bits) {
+ const ASTContext &Context = CGM.getContext();
+
+ QualType ETy = Context.getBaseElementType(ATy);
+ int Size = Context.getTypeSizeInChars(ETy).getQuantity();
+ SmallVector<uint64_t, 4> TmpBits(Size);
+ setUsedBits(CGM, ETy, 0, TmpBits);
+
+ for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) {
+ auto Src = TmpBits.begin();
+ auto Dst = Bits.begin() + Offset + I * Size;
+ for (int J = 0; J < Size; ++J)
+ *Dst++ |= *Src++;
+ }
+}
+
+// Set the bits in `Bits`, which correspond to the value representations of
+// the type `QTy`.
+static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
+ SmallVectorImpl<uint64_t> &Bits) {
+ if (const auto *RTy = QTy->getAs<RecordType>())
+ return setUsedBits(CGM, RTy, Offset, Bits);
+
+ ASTContext &Context = CGM.getContext();
+ if (const auto *ATy = Context.getAsConstantArrayType(QTy))
+ return setUsedBits(CGM, ATy, Offset, Bits);
+
+ int Size = Context.getTypeSizeInChars(QTy).getQuantity();
+ if (Size <= 0)
+ return;
+
+ std::fill_n(Bits.begin() + Offset, Size,
+ (uint64_t(1) << Context.getCharWidth()) - 1);
+}
+
+static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
+ int Pos, int Size, int CharWidth,
+ bool BigEndian) {
+ assert(Size > 0);
+ uint64_t Mask = 0;
+ if (BigEndian) {
+ for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
+ ++P)
+ Mask = (Mask << CharWidth) | *P;
+ } else {
+ auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
+ do
+ Mask = (Mask << CharWidth) | *--P;
+ while (P != End);
+ }
+ return Mask;
+}
+
+// Emit code to clear the bits in a record, which aren't a part of any user
+// declared member, when the record is a function return.
+llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
+ llvm::IntegerType *ITy,
+ QualType QTy) {
+ assert(Src->getType() == ITy);
+ assert(ITy->getScalarSizeInBits() <= 64);
+
+ const llvm::DataLayout &DataLayout = CGM.getDataLayout();
+ int Size = DataLayout.getTypeStoreSize(ITy);
+ SmallVector<uint64_t, 4> Bits(Size);
+ setUsedBits(CGM, QTy->getAs<RecordType>(), 0, Bits);
+
+ int CharWidth = CGM.getContext().getCharWidth();
+ uint64_t Mask =
+ buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
+
+ return Builder.CreateAnd(Src, Mask, "cmse.clear");
+}
+
+// Emit code to clear the bits in a record, which aren't a part of any user
+// declared member, when the record is a function argument.
+llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
+ llvm::ArrayType *ATy,
+ QualType QTy) {
+ const llvm::DataLayout &DataLayout = CGM.getDataLayout();
+ int Size = DataLayout.getTypeStoreSize(ATy);
+ SmallVector<uint64_t, 16> Bits(Size);
+ setUsedBits(CGM, QTy->getAs<RecordType>(), 0, Bits);
+
+ // Clear each element of the LLVM array.
+ int CharWidth = CGM.getContext().getCharWidth();
+ int CharsPerElt =
+ ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
+ int MaskIndex = 0;
+ llvm::Value *R = llvm::UndefValue::get(ATy);
+ for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
+ uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
+ DataLayout.isBigEndian());
+ MaskIndex += CharsPerElt;
+ llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
+ llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
+ R = Builder.CreateInsertValue(R, T1, I);
+ }
+
+ return R;
+}
+
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
bool EmitRetDbgLoc,
SourceLocation EndLoc) {
@@ -2991,6 +3292,14 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
llvm::Instruction *Ret;
if (RV) {
+ if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
+ // For certain return types, clear padding bits, as they may reveal
+ // sensitive information.
+ // Small struct/union types are passed as integers.
+ auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
+ if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType()))
+ RV = EmitCMSEClearRecord(RV, ITy, RetTy);
+ }
EmitReturnValueCheck(RV);
Ret = Builder.CreateRet(RV);
} else {
@@ -3006,6 +3315,11 @@ void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
if (!CurCodeDecl)
return;
+ // If the return block isn't reachable, neither is this check, so don't emit
+ // it.
+ if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
+ return;
+
ReturnsNonNullAttr *RetNNAttr = nullptr;
if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
@@ -3026,7 +3340,7 @@ void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
} else {
if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
if (auto *TSI = DD->getTypeSourceInfo())
- if (auto FTL = TSI->getTypeLoc().castAs<FunctionTypeLoc>())
+ if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
CheckKind = SanitizerKind::NullabilityReturn;
Handler = SanitizerHandler::NullabilityReturn;
@@ -3811,6 +4125,110 @@ void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
DeferredReplacements.push_back(std::make_pair(Old, New));
}
+namespace {
+
+/// Specify given \p NewAlign as the alignment of return value attribute. If
+/// such attribute already exists, re-set it to the maximal one of two options.
+LLVM_NODISCARD llvm::AttributeList
+maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
+ const llvm::AttributeList &Attrs,
+ llvm::Align NewAlign) {
+ llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
+ if (CurAlign >= NewAlign)
+ return Attrs;
+ llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
+ return Attrs
+ .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex,
+ llvm::Attribute::AttrKind::Alignment)
+ .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr);
+}
+
+template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
+protected:
+ CodeGenFunction &CGF;
+
+ /// We do nothing if this is, or becomes, nullptr.
+ const AlignedAttrTy *AA = nullptr;
+
+ llvm::Value *Alignment = nullptr; // May or may not be a constant.
+ llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
+
+ AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
+ : CGF(CGF_) {
+ if (!FuncDecl)
+ return;
+ AA = FuncDecl->getAttr<AlignedAttrTy>();
+ }
+
+public:
+ /// If we can, materialize the alignment as an attribute on return value.
+ LLVM_NODISCARD llvm::AttributeList
+ TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
+ if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment))
+ return Attrs;
+ const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
+ if (!AlignmentCI)
+ return Attrs;
+ // We may legitimately have non-power-of-2 alignment here.
+ // If so, this is UB land, emit it via `@llvm.assume` instead.
+ if (!AlignmentCI->getValue().isPowerOf2())
+ return Attrs;
+ llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
+ CGF.getLLVMContext(), Attrs,
+ llvm::Align(
+ AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
+ AA = nullptr; // We're done. Disallow doing anything else.
+ return NewAttrs;
+ }
+
+ /// Emit alignment assumption.
+ /// This is a general fallback that we take if either there is an offset,
+ /// or the alignment is variable or we are sanitizing for alignment.
+ void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
+ if (!AA)
+ return;
+ CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
+ AA->getLocation(), Alignment, OffsetCI);
+ AA = nullptr; // We're done. Disallow doing anything else.
+ }
+};
+
+/// Helper data structure to emit `AssumeAlignedAttr`.
+class AssumeAlignedAttrEmitter final
+ : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
+public:
+ AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
+ : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
+ if (!AA)
+ return;
+ // It is guaranteed that the alignment/offset are constants.
+ Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
+ if (Expr *Offset = AA->getOffset()) {
+ OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
+ if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
+ OffsetCI = nullptr;
+ }
+ }
+};
+
+/// Helper data structure to emit `AllocAlignAttr`.
+class AllocAlignAttrEmitter final
+ : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
+public:
+ AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
+ const CallArgList &CallArgs)
+ : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
+ if (!AA)
+ return;
+ // Alignment may or may not be a constant, and that is okay.
+ Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
+ .getRValue(CGF)
+ .getScalarVal();
+ }
+};
+
+} // namespace
+
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
const CGCallee &Callee,
ReturnValueSlot ReturnValue,
@@ -3829,7 +4247,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
- if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
// We can only guarantee that a function is called from the correct
// context/function based on the appropriate target attributes,
// so only check in the case where we have both always_inline and target
@@ -3840,6 +4258,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
TargetDecl->hasAttr<TargetAttr>())
checkTargetFeatures(Loc, FD);
+ // Some architectures (such as x86-64) have the ABI changed based on
+ // attribute-target/features. Give them a chance to diagnose.
+ CGM.getTargetCodeGenInfo().checkFunctionCallABI(
+ CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
+ }
+
#ifndef NDEBUG
if (!(CallInfo.isVariadic() && CallInfo.getArgStruct())) {
// For an inalloca varargs function, we don't expect CallInfo to match the
@@ -3940,18 +4364,39 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
assert(NumIRArgs == 0);
assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
if (I->isAggregate()) {
- // Replace the placeholder with the appropriate argument slot GEP.
Address Addr = I->hasLValue()
? I->getKnownLValue().getAddress(*this)
: I->getKnownRValue().getAggregateAddress();
llvm::Instruction *Placeholder =
cast<llvm::Instruction>(Addr.getPointer());
- CGBuilderTy::InsertPoint IP = Builder.saveIP();
- Builder.SetInsertPoint(Placeholder);
- Addr =
- Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
- Builder.restoreIP(IP);
+
+ if (!ArgInfo.getInAllocaIndirect()) {
+ // Replace the placeholder with the appropriate argument slot GEP.
+ CGBuilderTy::InsertPoint IP = Builder.saveIP();
+ Builder.SetInsertPoint(Placeholder);
+ Addr = Builder.CreateStructGEP(ArgMemory,
+ ArgInfo.getInAllocaFieldIndex());
+ Builder.restoreIP(IP);
+ } else {
+ // For indirect things such as overaligned structs, replace the
+ // placeholder with a regular aggregate temporary alloca. Store the
+ // address of this alloca into the struct.
+ Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
+ Address ArgSlot = Builder.CreateStructGEP(
+ ArgMemory, ArgInfo.getInAllocaFieldIndex());
+ Builder.CreateStore(Addr.getPointer(), ArgSlot);
+ }
deferPlaceholderReplacement(Placeholder, Addr.getPointer());
+ } else if (ArgInfo.getInAllocaIndirect()) {
+ // Make a temporary alloca and store the address of it into the argument
+ // struct.
+ Address Addr = CreateMemTempWithoutCast(
+ I->Ty, getContext().getTypeAlignInChars(I->Ty),
+ "indirect-arg-temp");
+ I->copyInto(*this, Addr);
+ Address ArgSlot =
+ Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
+ Builder.CreateStore(Addr.getPointer(), ArgSlot);
} else {
// Store the RValue into the argument struct.
Address Addr =
@@ -4001,8 +4446,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
bool NeedCopy = false;
if (Addr.getAlignment() < Align &&
- llvm::getOrEnforceKnownAlignment(V, Align.getQuantity(), *TD) <
- Align.getQuantity()) {
+ llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
+ Align.getAsAlign()) {
NeedCopy = true;
} else if (I->hasLValue()) {
auto LV = I->getKnownLValue();
@@ -4128,7 +4573,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
llvm::StructType *STy =
dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
- llvm::Type *SrcTy = Src.getType()->getElementType();
+ llvm::Type *SrcTy = Src.getElementType();
uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
@@ -4156,8 +4601,18 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
} else {
// In the simple case, just pass the coerced loaded value.
assert(NumIRArgs == 1);
- IRCallArgs[FirstIRArg] =
- CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
+ llvm::Value *Load =
+ CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
+
+ if (CallInfo.isCmseNSCall()) {
+ // For certain parameter types, clear padding bits, as they may reveal
+ // sensitive information.
+ // Small struct/union types are passed as integer arrays.
+ auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
+ if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))
+ Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
+ }
+ IRCallArgs[FirstIRArg] = Load;
}
break;
@@ -4328,8 +4783,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// Update the largest vector width if any arguments have vector types.
for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType()))
- LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
- VT->getPrimitiveSizeInBits().getFixedSize());
+ LargestVectorWidth =
+ std::max((uint64_t)LargestVectorWidth,
+ VT->getPrimitiveSizeInBits().getKnownMinSize());
}
// Compute the calling convention and attributes.
@@ -4346,6 +4802,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
llvm::Attribute::StrictFP);
+ // Add call-site nomerge attribute if exists.
+ if (InNoMergeAttributedStmt)
+ Attrs =
+ Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
+ llvm::Attribute::NoMerge);
+
// Apply some call-site-specific attributes.
// TODO: work this into building the attribute set.
@@ -4378,8 +4840,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
CannotThrow = true;
} else {
// Otherwise, nounwind call sites will never throw.
- CannotThrow = Attrs.hasAttribute(llvm::AttributeList::FunctionIndex,
- llvm::Attribute::NoUnwind);
+ CannotThrow = Attrs.hasFnAttribute(llvm::Attribute::NoUnwind);
}
// If we made a temporary, be sure to clean up after ourselves. Note that we
@@ -4402,6 +4863,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
llvm::Attribute::StrictFP);
+ AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
+ Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
+
+ AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
+ Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
+
// Emit the actual call/invoke instruction.
llvm::CallBase *CI;
if (!InvokeDest) {
@@ -4437,8 +4904,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// Update largest vector width from the return type.
if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType()))
- LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
- VT->getPrimitiveSizeInBits().getFixedSize());
+ LargestVectorWidth =
+ std::max((uint64_t)LargestVectorWidth,
+ VT->getPrimitiveSizeInBits().getKnownMinSize());
// Insert instrumentation or attach profile metadata at indirect call sites.
// For more details, see the comment before the definition of
@@ -4461,7 +4929,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// Add metadata for calls to MSAllocator functions
if (getDebugInfo() && TargetDecl &&
TargetDecl->hasAttr<MSAllocatorAttr>())
- getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy, Loc);
+ getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy->getPointeeType(), Loc);
// 4. Finish the call.
@@ -4581,7 +5049,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
DestPtr = CreateMemTemp(RetTy, "agg.tmp");
DestIsVolatile = false;
}
- BuildAggStore(*this, CI, DestPtr, DestIsVolatile);
+ EmitAggregateStore(CI, DestPtr, DestIsVolatile);
return RValue::getAggregate(DestPtr);
}
case TEK_Scalar: {
@@ -4620,22 +5088,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// Emit the assume_aligned check on the return value.
if (Ret.isScalar() && TargetDecl) {
- if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) {
- llvm::Value *OffsetValue = nullptr;
- if (const auto *Offset = AA->getOffset())
- OffsetValue = EmitScalarExpr(Offset);
-
- llvm::Value *Alignment = EmitScalarExpr(AA->getAlignment());
- llvm::ConstantInt *AlignmentCI = cast<llvm::ConstantInt>(Alignment);
- EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, AA->getLocation(),
- AlignmentCI, OffsetValue);
- } else if (const auto *AA = TargetDecl->getAttr<AllocAlignAttr>()) {
- llvm::Value *AlignmentVal = CallArgs[AA->getParamIndex().getLLVMIndex()]
- .getRValue(*this)
- .getScalarVal();
- EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, AA->getLocation(),
- AlignmentVal);
- }
+ AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
+ AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
}
// Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
@@ -4643,6 +5097,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
LifetimeEnd.Emit(*this, /*Flags=*/{});
+ if (!ReturnValue.isExternallyDestructed() &&
+ RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
+ pushDestroy(QualType::DK_nontrivial_c_struct, Ret.getAggregateAddress(),
+ RetTy);
+
return Ret;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-06 01:28:46 +0000