aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Utils/CallPromotionUtils.cpp')
-rw-r--r--llvm/lib/Transforms/Utils/CallPromotionUtils.cpp231
1 files changed, 184 insertions, 47 deletions
diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
index f04d76e70c0d..5a47c1fd0b6c 100644
--- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
@@ -12,7 +12,10 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
+#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/TypeMetadataUtils.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
using namespace llvm;
@@ -158,32 +161,31 @@ static void createRetPHINode(Instruction *OrigInst, Instruction *NewInst,
/// %t1 = bitcast i32 %t0 to ...
/// br label %normal_dst
///
-static void createRetBitCast(CallSite CS, Type *RetTy, CastInst **RetBitCast) {
+static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) {
// Save the users of the calling instruction. These uses will be changed to
// use the bitcast after we create it.
SmallVector<User *, 16> UsersToUpdate;
- for (User *U : CS.getInstruction()->users())
+ for (User *U : CB.users())
UsersToUpdate.push_back(U);
// Determine an appropriate location to create the bitcast for the return
// value. The location depends on if we have a call or invoke instruction.
Instruction *InsertBefore = nullptr;
- if (auto *Invoke = dyn_cast<InvokeInst>(CS.getInstruction()))
+ if (auto *Invoke = dyn_cast<InvokeInst>(&CB))
InsertBefore =
&SplitEdge(Invoke->getParent(), Invoke->getNormalDest())->front();
else
- InsertBefore = &*std::next(CS.getInstruction()->getIterator());
+ InsertBefore = &*std::next(CB.getIterator());
// Bitcast the return value to the correct type.
- auto *Cast = CastInst::CreateBitOrPointerCast(CS.getInstruction(), RetTy, "",
- InsertBefore);
+ auto *Cast = CastInst::CreateBitOrPointerCast(&CB, RetTy, "", InsertBefore);
if (RetBitCast)
*RetBitCast = Cast;
// Replace all the original uses of the calling instruction with the bitcast.
for (User *U : UsersToUpdate)
- U->replaceUsesOfWith(CS.getInstruction(), Cast);
+ U->replaceUsesOfWith(&CB, Cast);
}
/// Predicate and clone the given call site.
@@ -253,26 +255,91 @@ static void createRetBitCast(CallSite CS, Type *RetTy, CastInst **RetBitCast) {
/// %t2 = phi i32 [ %t0, %else_bb ], [ %t1, %then_bb ]
/// br %normal_dst
///
-static Instruction *versionCallSite(CallSite CS, Value *Callee,
- MDNode *BranchWeights) {
-
- IRBuilder<> Builder(CS.getInstruction());
- Instruction *OrigInst = CS.getInstruction();
+/// An indirect musttail call is processed slightly differently in that:
+/// 1. No merge block needed for the orginal and the cloned callsite, since
+/// either one ends the flow. No phi node is needed either.
+/// 2. The return statement following the original call site is duplicated too
+/// and placed immediately after the cloned call site per the IR convention.
+///
+/// For example, the musttail call instruction below:
+///
+/// orig_bb:
+/// %t0 = musttail call i32 %ptr()
+/// ...
+///
+/// Is replaced by the following:
+///
+/// cond_bb:
+/// %cond = icmp eq i32 ()* %ptr, @func
+/// br i1 %cond, %then_bb, %orig_bb
+///
+/// then_bb:
+/// ; The clone of the original call instruction is placed in the "then"
+/// ; block. It is not yet promoted.
+/// %t1 = musttail call i32 %ptr()
+/// ret %t1
+///
+/// orig_bb:
+/// ; The original call instruction stays in its original block.
+/// %t0 = musttail call i32 %ptr()
+/// ret %t0
+static CallBase &versionCallSite(CallBase &CB, Value *Callee,
+ MDNode *BranchWeights) {
+
+ IRBuilder<> Builder(&CB);
+ CallBase *OrigInst = &CB;
BasicBlock *OrigBlock = OrigInst->getParent();
// Create the compare. The called value and callee must have the same type to
// be compared.
- if (CS.getCalledValue()->getType() != Callee->getType())
- Callee = Builder.CreateBitCast(Callee, CS.getCalledValue()->getType());
- auto *Cond = Builder.CreateICmpEQ(CS.getCalledValue(), Callee);
+ if (CB.getCalledOperand()->getType() != Callee->getType())
+ Callee = Builder.CreateBitCast(Callee, CB.getCalledOperand()->getType());
+ auto *Cond = Builder.CreateICmpEQ(CB.getCalledOperand(), Callee);
+
+ if (OrigInst->isMustTailCall()) {
+ // Create an if-then structure. The original instruction stays in its block,
+ // and a clone of the original instruction is placed in the "then" block.
+ Instruction *ThenTerm =
+ SplitBlockAndInsertIfThen(Cond, &CB, false, BranchWeights);
+ BasicBlock *ThenBlock = ThenTerm->getParent();
+ ThenBlock->setName("if.true.direct_targ");
+ CallBase *NewInst = cast<CallBase>(OrigInst->clone());
+ NewInst->insertBefore(ThenTerm);
+
+ // Place a clone of the optional bitcast after the new call site.
+ Value *NewRetVal = NewInst;
+ auto Next = OrigInst->getNextNode();
+ if (auto *BitCast = dyn_cast_or_null<BitCastInst>(Next)) {
+ assert(BitCast->getOperand(0) == OrigInst &&
+ "bitcast following musttail call must use the call");
+ auto NewBitCast = BitCast->clone();
+ NewBitCast->replaceUsesOfWith(OrigInst, NewInst);
+ NewBitCast->insertBefore(ThenTerm);
+ NewRetVal = NewBitCast;
+ Next = BitCast->getNextNode();
+ }
+
+ // Place a clone of the return instruction after the new call site.
+ ReturnInst *Ret = dyn_cast_or_null<ReturnInst>(Next);
+ assert(Ret && "musttail call must precede a ret with an optional bitcast");
+ auto NewRet = Ret->clone();
+ if (Ret->getReturnValue())
+ NewRet->replaceUsesOfWith(Ret->getReturnValue(), NewRetVal);
+ NewRet->insertBefore(ThenTerm);
+
+ // A return instructions is terminating, so we don't need the terminator
+ // instruction just created.
+ ThenTerm->eraseFromParent();
+
+ return *NewInst;
+ }
// Create an if-then-else structure. The original instruction is moved into
// the "else" block, and a clone of the original instruction is placed in the
// "then" block.
Instruction *ThenTerm = nullptr;
Instruction *ElseTerm = nullptr;
- SplitBlockAndInsertIfThenElse(Cond, CS.getInstruction(), &ThenTerm, &ElseTerm,
- BranchWeights);
+ SplitBlockAndInsertIfThenElse(Cond, &CB, &ThenTerm, &ElseTerm, BranchWeights);
BasicBlock *ThenBlock = ThenTerm->getParent();
BasicBlock *ElseBlock = ElseTerm->getParent();
BasicBlock *MergeBlock = OrigInst->getParent();
@@ -281,7 +348,7 @@ static Instruction *versionCallSite(CallSite CS, Value *Callee,
ElseBlock->setName("if.false.orig_indirect");
MergeBlock->setName("if.end.icp");
- Instruction *NewInst = OrigInst->clone();
+ CallBase *NewInst = cast<CallBase>(OrigInst->clone());
OrigInst->moveBefore(ElseTerm);
NewInst->insertBefore(ThenTerm);
@@ -313,18 +380,18 @@ static Instruction *versionCallSite(CallSite CS, Value *Callee,
// Create a phi node for the returned value of the call site.
createRetPHINode(OrigInst, NewInst, MergeBlock, Builder);
- return NewInst;
+ return *NewInst;
}
-bool llvm::isLegalToPromote(CallSite CS, Function *Callee,
+bool llvm::isLegalToPromote(const CallBase &CB, Function *Callee,
const char **FailureReason) {
- assert(!CS.getCalledFunction() && "Only indirect call sites can be promoted");
+ assert(!CB.getCalledFunction() && "Only indirect call sites can be promoted");
auto &DL = Callee->getParent()->getDataLayout();
// Check the return type. The callee's return value type must be bitcast
// compatible with the call site's type.
- Type *CallRetTy = CS.getInstruction()->getType();
+ Type *CallRetTy = CB.getType();
Type *FuncRetTy = Callee->getReturnType();
if (CallRetTy != FuncRetTy)
if (!CastInst::isBitOrNoopPointerCastable(FuncRetTy, CallRetTy, DL)) {
@@ -336,9 +403,12 @@ bool llvm::isLegalToPromote(CallSite CS, Function *Callee,
// The number of formal arguments of the callee.
unsigned NumParams = Callee->getFunctionType()->getNumParams();
+ // The number of actual arguments in the call.
+ unsigned NumArgs = CB.arg_size();
+
// Check the number of arguments. The callee and call site must agree on the
// number of arguments.
- if (CS.arg_size() != NumParams && !Callee->isVarArg()) {
+ if (NumArgs != NumParams && !Callee->isVarArg()) {
if (FailureReason)
*FailureReason = "The number of arguments mismatch";
return false;
@@ -347,9 +417,10 @@ bool llvm::isLegalToPromote(CallSite CS, Function *Callee,
// Check the argument types. The callee's formal argument types must be
// bitcast compatible with the corresponding actual argument types of the call
// site.
- for (unsigned I = 0; I < NumParams; ++I) {
+ unsigned I = 0;
+ for (; I < NumParams; ++I) {
Type *FormalTy = Callee->getFunctionType()->getFunctionParamType(I);
- Type *ActualTy = CS.getArgument(I)->getType();
+ Type *ActualTy = CB.getArgOperand(I)->getType();
if (FormalTy == ActualTy)
continue;
if (!CastInst::isBitOrNoopPointerCastable(ActualTy, FormalTy, DL)) {
@@ -358,35 +429,43 @@ bool llvm::isLegalToPromote(CallSite CS, Function *Callee,
return false;
}
}
+ for (; I < NumArgs; I++) {
+ // Vararg functions can have more arguments than paramters.
+ assert(Callee->isVarArg());
+ if (CB.paramHasAttr(I, Attribute::StructRet)) {
+ *FailureReason = "SRet arg to vararg function";
+ return false;
+ }
+ }
return true;
}
-Instruction *llvm::promoteCall(CallSite CS, Function *Callee,
- CastInst **RetBitCast) {
- assert(!CS.getCalledFunction() && "Only indirect call sites can be promoted");
+CallBase &llvm::promoteCall(CallBase &CB, Function *Callee,
+ CastInst **RetBitCast) {
+ assert(!CB.getCalledFunction() && "Only indirect call sites can be promoted");
// Set the called function of the call site to be the given callee (but don't
// change the type).
- cast<CallBase>(CS.getInstruction())->setCalledOperand(Callee);
+ CB.setCalledOperand(Callee);
// Since the call site will no longer be direct, we must clear metadata that
// is only appropriate for indirect calls. This includes !prof and !callees
// metadata.
- CS.getInstruction()->setMetadata(LLVMContext::MD_prof, nullptr);
- CS.getInstruction()->setMetadata(LLVMContext::MD_callees, nullptr);
+ CB.setMetadata(LLVMContext::MD_prof, nullptr);
+ CB.setMetadata(LLVMContext::MD_callees, nullptr);
// If the function type of the call site matches that of the callee, no
// additional work is required.
- if (CS.getFunctionType() == Callee->getFunctionType())
- return CS.getInstruction();
+ if (CB.getFunctionType() == Callee->getFunctionType())
+ return CB;
// Save the return types of the call site and callee.
- Type *CallSiteRetTy = CS.getInstruction()->getType();
+ Type *CallSiteRetTy = CB.getType();
Type *CalleeRetTy = Callee->getReturnType();
// Change the function type of the call site the match that of the callee.
- CS.mutateFunctionType(Callee->getFunctionType());
+ CB.mutateFunctionType(Callee->getFunctionType());
// Inspect the arguments of the call site. If an argument's type doesn't
// match the corresponding formal argument's type in the callee, bitcast it
@@ -395,19 +474,18 @@ Instruction *llvm::promoteCall(CallSite CS, Function *Callee,
auto CalleeParamNum = CalleeType->getNumParams();
LLVMContext &Ctx = Callee->getContext();
- const AttributeList &CallerPAL = CS.getAttributes();
+ const AttributeList &CallerPAL = CB.getAttributes();
// The new list of argument attributes.
SmallVector<AttributeSet, 4> NewArgAttrs;
bool AttributeChanged = false;
for (unsigned ArgNo = 0; ArgNo < CalleeParamNum; ++ArgNo) {
- auto *Arg = CS.getArgument(ArgNo);
+ auto *Arg = CB.getArgOperand(ArgNo);
Type *FormalTy = CalleeType->getParamType(ArgNo);
Type *ActualTy = Arg->getType();
if (FormalTy != ActualTy) {
- auto *Cast = CastInst::CreateBitOrPointerCast(Arg, FormalTy, "",
- CS.getInstruction());
- CS.setArgument(ArgNo, Cast);
+ auto *Cast = CastInst::CreateBitOrPointerCast(Arg, FormalTy, "", &CB);
+ CB.setArgOperand(ArgNo, Cast);
// Remove any incompatible attributes for the argument.
AttrBuilder ArgAttrs(CallerPAL.getParamAttributes(ArgNo));
@@ -432,30 +510,89 @@ Instruction *llvm::promoteCall(CallSite CS, Function *Callee,
// Remove any incompatible return value attribute.
AttrBuilder RAttrs(CallerPAL, AttributeList::ReturnIndex);
if (!CallSiteRetTy->isVoidTy() && CallSiteRetTy != CalleeRetTy) {
- createRetBitCast(CS, CallSiteRetTy, RetBitCast);
+ createRetBitCast(CB, CallSiteRetTy, RetBitCast);
RAttrs.remove(AttributeFuncs::typeIncompatible(CalleeRetTy));
AttributeChanged = true;
}
// Set the new callsite attribute.
if (AttributeChanged)
- CS.setAttributes(AttributeList::get(Ctx, CallerPAL.getFnAttributes(),
+ CB.setAttributes(AttributeList::get(Ctx, CallerPAL.getFnAttributes(),
AttributeSet::get(Ctx, RAttrs),
NewArgAttrs));
- return CS.getInstruction();
+ return CB;
}
-Instruction *llvm::promoteCallWithIfThenElse(CallSite CS, Function *Callee,
- MDNode *BranchWeights) {
+CallBase &llvm::promoteCallWithIfThenElse(CallBase &CB, Function *Callee,
+ MDNode *BranchWeights) {
// Version the indirect call site. If the called value is equal to the given
// callee, 'NewInst' will be executed, otherwise the original call site will
// be executed.
- Instruction *NewInst = versionCallSite(CS, Callee, BranchWeights);
+ CallBase &NewInst = versionCallSite(CB, Callee, BranchWeights);
// Promote 'NewInst' so that it directly calls the desired function.
- return promoteCall(CallSite(NewInst), Callee);
+ return promoteCall(NewInst, Callee);
+}
+
+bool llvm::tryPromoteCall(CallBase &CB) {
+ assert(!CB.getCalledFunction());
+ Module *M = CB.getCaller()->getParent();
+ const DataLayout &DL = M->getDataLayout();
+ Value *Callee = CB.getCalledOperand();
+
+ LoadInst *VTableEntryLoad = dyn_cast<LoadInst>(Callee);
+ if (!VTableEntryLoad)
+ return false; // Not a vtable entry load.
+ Value *VTableEntryPtr = VTableEntryLoad->getPointerOperand();
+ APInt VTableOffset(DL.getTypeSizeInBits(VTableEntryPtr->getType()), 0);
+ Value *VTableBasePtr = VTableEntryPtr->stripAndAccumulateConstantOffsets(
+ DL, VTableOffset, /* AllowNonInbounds */ true);
+ LoadInst *VTablePtrLoad = dyn_cast<LoadInst>(VTableBasePtr);
+ if (!VTablePtrLoad)
+ return false; // Not a vtable load.
+ Value *Object = VTablePtrLoad->getPointerOperand();
+ APInt ObjectOffset(DL.getTypeSizeInBits(Object->getType()), 0);
+ Value *ObjectBase = Object->stripAndAccumulateConstantOffsets(
+ DL, ObjectOffset, /* AllowNonInbounds */ true);
+ if (!(isa<AllocaInst>(ObjectBase) && ObjectOffset == 0))
+ // Not an Alloca or the offset isn't zero.
+ return false;
+
+ // Look for the vtable pointer store into the object by the ctor.
+ BasicBlock::iterator BBI(VTablePtrLoad);
+ Value *VTablePtr = FindAvailableLoadedValue(
+ VTablePtrLoad, VTablePtrLoad->getParent(), BBI, 0, nullptr, nullptr);
+ if (!VTablePtr)
+ return false; // No vtable found.
+ APInt VTableOffsetGVBase(DL.getTypeSizeInBits(VTablePtr->getType()), 0);
+ Value *VTableGVBase = VTablePtr->stripAndAccumulateConstantOffsets(
+ DL, VTableOffsetGVBase, /* AllowNonInbounds */ true);
+ GlobalVariable *GV = dyn_cast<GlobalVariable>(VTableGVBase);
+ if (!(GV && GV->isConstant() && GV->hasDefinitiveInitializer()))
+ // Not in the form of a global constant variable with an initializer.
+ return false;
+
+ Constant *VTableGVInitializer = GV->getInitializer();
+ APInt VTableGVOffset = VTableOffsetGVBase + VTableOffset;
+ if (!(VTableGVOffset.getActiveBits() <= 64))
+ return false; // Out of range.
+ Constant *Ptr = getPointerAtOffset(VTableGVInitializer,
+ VTableGVOffset.getZExtValue(),
+ *M);
+ if (!Ptr)
+ return false; // No constant (function) pointer found.
+ Function *DirectCallee = dyn_cast<Function>(Ptr->stripPointerCasts());
+ if (!DirectCallee)
+ return false; // No function pointer found.
+
+ if (!isLegalToPromote(CB, DirectCallee))
+ return false;
+
+ // Success.
+ promoteCall(CB, DirectCallee);
+ return true;
}
#undef DEBUG_TYPE
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-04 20:10:23 +0000