aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp')
-rw-r--r--llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp432
1 files changed, 328 insertions, 104 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index a7b1953ce81c..d3282779d9f5 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -6,7 +6,7 @@
//
//===----------------------------------------------------------------------===//
//
-// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
+// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
// It also builds the data structures and initialization code needed for
// updating execution counts and emitting the profile at runtime.
//
@@ -14,6 +14,7 @@
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
@@ -23,6 +24,7 @@
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DIBuilder.h"
@@ -47,6 +49,9 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#include <algorithm>
@@ -190,7 +195,8 @@ public:
auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
- Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
+ Addr = Builder.CreateIntToPtr(BiasInst,
+ PointerType::getUnqual(Ty->getContext()));
}
if (AtomicCounterUpdatePromoted)
// automic update currently can only be promoted across the current
@@ -241,7 +247,10 @@ public:
return;
for (BasicBlock *ExitBlock : LoopExitBlocks) {
- if (BlockSet.insert(ExitBlock).second) {
+ if (BlockSet.insert(ExitBlock).second &&
+ llvm::none_of(predecessors(ExitBlock), [&](const BasicBlock *Pred) {
+ return llvm::isPresplitCoroSuspendExitEdge(*Pred, *ExitBlock);
+ })) {
ExitBlocks.push_back(ExitBlock);
InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
}
@@ -430,6 +439,15 @@ bool InstrProfiling::lowerIntrinsics(Function *F) {
} else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
lowerValueProfileInst(IPVP);
MadeChange = true;
+ } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
+ IPMP->eraseFromParent();
+ MadeChange = true;
+ } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
+ lowerMCDCTestVectorBitmapUpdate(IPBU);
+ MadeChange = true;
+ } else if (auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
+ lowerMCDCCondBitmapUpdate(IPTU);
+ MadeChange = true;
}
}
}
@@ -544,19 +562,27 @@ bool InstrProfiling::run(
// the instrumented function. This is counting the number of instrumented
// target value sites to enter it as field in the profile data variable.
for (Function &F : M) {
- InstrProfInstBase *FirstProfInst = nullptr;
- for (BasicBlock &BB : F)
- for (auto I = BB.begin(), E = BB.end(); I != E; I++)
+ InstrProfCntrInstBase *FirstProfInst = nullptr;
+ for (BasicBlock &BB : F) {
+ for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
computeNumValueSiteCounts(Ind);
- else if (FirstProfInst == nullptr &&
- (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
- FirstProfInst = dyn_cast<InstrProfInstBase>(I);
+ else {
+ if (FirstProfInst == nullptr &&
+ (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
+ FirstProfInst = dyn_cast<InstrProfCntrInstBase>(I);
+ // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
+ if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(I))
+ static_cast<void>(getOrCreateRegionBitmaps(Params));
+ }
+ }
+ }
- // Value profiling intrinsic lowering requires per-function profile data
- // variable to be created first.
- if (FirstProfInst != nullptr)
+ // Use a profile intrinsic to create the region counters and data variable.
+ // Also create the data variable based on the MCDCParams.
+ if (FirstProfInst != nullptr) {
static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
+ }
}
for (Function &F : M)
@@ -651,15 +677,11 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
SmallVector<OperandBundleDef, 1> OpBundles;
Ind->getOperandBundlesAsDefs(OpBundles);
if (!IsMemOpSize) {
- Value *Args[3] = {Ind->getTargetValue(),
- Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
- Builder.getInt32(Index)};
+ Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
OpBundles);
} else {
- Value *Args[3] = {Ind->getTargetValue(),
- Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
- Builder.getInt32(Index)};
+ Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
Call = Builder.CreateCall(
getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
Args, OpBundles);
@@ -670,7 +692,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Ind->eraseFromParent();
}
-Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
+Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) {
auto *Counters = getOrCreateRegionCounters(I);
IRBuilder<> Builder(I);
@@ -710,6 +732,25 @@ Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
return Builder.CreateIntToPtr(Add, Addr->getType());
}
+Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
+ auto *Bitmaps = getOrCreateRegionBitmaps(I);
+ IRBuilder<> Builder(I);
+
+ auto *Addr = Builder.CreateConstInBoundsGEP2_32(
+ Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue());
+
+ if (isRuntimeCounterRelocationEnabled()) {
+ LLVMContext &Ctx = M->getContext();
+ Ctx.diagnose(DiagnosticInfoPGOProfile(
+ M->getName().data(),
+ Twine("Runtime counter relocation is presently not supported for MC/DC "
+ "bitmaps."),
+ DS_Warning));
+ }
+
+ return Addr;
+}
+
void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
auto *Addr = getCounterAddress(CoverInstruction);
IRBuilder<> Builder(CoverInstruction);
@@ -769,6 +810,86 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
CoverageNamesVar->eraseFromParent();
}
+void InstrProfiling::lowerMCDCTestVectorBitmapUpdate(
+ InstrProfMCDCTVBitmapUpdate *Update) {
+ IRBuilder<> Builder(Update);
+ auto *Int8Ty = Type::getInt8Ty(M->getContext());
+ auto *Int8PtrTy = PointerType::getUnqual(M->getContext());
+ auto *Int32Ty = Type::getInt32Ty(M->getContext());
+ auto *Int64Ty = Type::getInt64Ty(M->getContext());
+ auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
+ auto *BitmapAddr = getBitmapAddress(Update);
+
+ // Load Temp Val.
+ // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
+ auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
+
+ // Calculate byte offset using div8.
+ // %1 = lshr i32 %mcdc.temp, 3
+ auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
+
+ // Add byte offset to section base byte address.
+ // %2 = zext i32 %1 to i64
+ // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2
+ auto *BitmapByteAddr =
+ Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty),
+ Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty));
+
+ // Convert to a pointer.
+ // %4 = inttoptr i32 %3 to ptr
+ BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy);
+
+ // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
+ // %5 = and i32 %mcdc.temp, 7
+ // %6 = trunc i32 %5 to i8
+ auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
+
+ // Shift bit offset left to form a bitmap.
+ // %7 = shl i8 1, %6
+ auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
+
+ // Load profile bitmap byte.
+ // %mcdc.bits = load i8, ptr %4, align 1
+ auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");
+
+ // Perform logical OR of profile bitmap byte and shifted bit offset.
+ // %8 = or i8 %mcdc.bits, %7
+ auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);
+
+ // Store the updated profile bitmap byte.
+ // store i8 %8, ptr %3, align 1
+ Builder.CreateStore(Result, BitmapByteAddr);
+ Update->eraseFromParent();
+}
+
+void InstrProfiling::lowerMCDCCondBitmapUpdate(
+ InstrProfMCDCCondBitmapUpdate *Update) {
+ IRBuilder<> Builder(Update);
+ auto *Int32Ty = Type::getInt32Ty(M->getContext());
+ auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
+
+ // Load the MCDC temporary value from the stack.
+ // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
+ auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
+
+ // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits.
+ // %1 = zext i1 %tobool to i32
+ auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty);
+
+ // Shift the boolean value left (by the condition's ID) to form a bitmap.
+ // %2 = shl i32 %1, <Update->getCondID()>
+ auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID());
+
+ // Perform logical OR of the bitmap against the loaded MCDC temporary value.
+ // %3 = or i32 %mcdc.temp, %2
+ auto *Result = Builder.CreateOr(Temp, ShiftedVal);
+
+ // Store the updated temporary value back to the stack.
+ // store i32 %3, ptr %mcdc.addr, align 4
+ Builder.CreateStore(Result, MCDCCondBitmapAddr);
+ Update->eraseFromParent();
+}
+
/// Get the name of a profiling variable for a particular function.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
bool &Renamed) {
@@ -784,7 +905,7 @@ static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
Renamed = true;
uint64_t FuncHash = Inc->getHash()->getZExtValue();
SmallVector<char, 24> HashPostfix;
- if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
+ if (Name.ends_with((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
return (Prefix + Name).str();
return (Prefix + Name + "." + Twine(FuncHash)).str();
}
@@ -878,7 +999,7 @@ static inline bool shouldUsePublicSymbol(Function *Fn) {
}
static inline Constant *getFuncAddrForProfData(Function *Fn) {
- auto *Int8PtrTy = Type::getInt8PtrTy(Fn->getContext());
+ auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());
// Store a nullptr in __llvm_profd, if we shouldn't use a real address
if (!shouldRecordFunctionAddr(Fn))
return ConstantPointerNull::get(Int8PtrTy);
@@ -886,7 +1007,7 @@ static inline Constant *getFuncAddrForProfData(Function *Fn) {
// If we can't use an alias, we must use the public symbol, even though this
// may require a symbolic relocation.
if (shouldUsePublicSymbol(Fn))
- return ConstantExpr::getBitCast(Fn, Int8PtrTy);
+ return Fn;
// When possible use a private alias to avoid symbolic relocations.
auto *GA = GlobalAlias::create(GlobalValue::LinkageTypes::PrivateLinkage,
@@ -909,7 +1030,7 @@ static inline Constant *getFuncAddrForProfData(Function *Fn) {
// appendToCompilerUsed(*Fn->getParent(), {GA});
- return ConstantExpr::getBitCast(GA, Int8PtrTy);
+ return GA;
}
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
@@ -924,37 +1045,31 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
return true;
}
-GlobalVariable *
-InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
- GlobalValue::LinkageTypes Linkage) {
- uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
- auto &Ctx = M->getContext();
- GlobalVariable *GV;
- if (isa<InstrProfCoverInst>(Inc)) {
- auto *CounterTy = Type::getInt8Ty(Ctx);
- auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
- // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
- std::vector<Constant *> InitialValues(NumCounters,
- Constant::getAllOnesValue(CounterTy));
- GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
- ConstantArray::get(CounterArrTy, InitialValues),
- Name);
- GV->setAlignment(Align(1));
- } else {
- auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
- GV = new GlobalVariable(*M, CounterTy, false, Linkage,
- Constant::getNullValue(CounterTy), Name);
- GV->setAlignment(Align(8));
- }
- return GV;
+void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn,
+ StringRef VarName) {
+ bool DataReferencedByCode = profDataReferencedByCode(*M);
+ bool NeedComdat = needsComdatForCounter(*Fn, *M);
+ bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
+
+ if (!UseComdat)
+ return;
+
+ StringRef GroupName =
+ TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName;
+ Comdat *C = M->getOrInsertComdat(GroupName);
+ if (!NeedComdat)
+ C->setSelectionKind(Comdat::NoDeduplicate);
+ GV->setComdat(C);
+ // COFF doesn't allow the comdat group leader to have private linkage, so
+ // upgrade private linkage to internal linkage to produce a symbol table
+ // entry.
+ if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
+ GV->setLinkage(GlobalValue::InternalLinkage);
}
-GlobalVariable *
-InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
+GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc,
+ InstrProfSectKind IPSK) {
GlobalVariable *NamePtr = Inc->getName();
- auto &PD = ProfileDataMap[NamePtr];
- if (PD.RegionCounters)
- return PD.RegionCounters;
// Match the linkage and visibility of the name global.
Function *Fn = Inc->getParent()->getParent();
@@ -993,42 +1108,101 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
// nodeduplicate COMDAT which is lowered to a zero-flag section group. This
// allows -z start-stop-gc to discard the entire group when the function is
// discarded.
- bool DataReferencedByCode = profDataReferencedByCode(*M);
- bool NeedComdat = needsComdatForCounter(*Fn, *M);
bool Renamed;
- std::string CntsVarName =
- getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
- std::string DataVarName =
- getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
- auto MaybeSetComdat = [&](GlobalVariable *GV) {
- bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
- if (UseComdat) {
- StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
- ? GV->getName()
- : CntsVarName;
- Comdat *C = M->getOrInsertComdat(GroupName);
- if (!NeedComdat)
- C->setSelectionKind(Comdat::NoDeduplicate);
- GV->setComdat(C);
- // COFF doesn't allow the comdat group leader to have private linkage, so
- // upgrade private linkage to internal linkage to produce a symbol table
- // entry.
- if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
- GV->setLinkage(GlobalValue::InternalLinkage);
- }
- };
+ GlobalVariable *Ptr;
+ StringRef VarPrefix;
+ std::string VarName;
+ if (IPSK == IPSK_cnts) {
+ VarPrefix = getInstrProfCountersVarPrefix();
+ VarName = getVarName(Inc, VarPrefix, Renamed);
+ InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Inc);
+ Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
+ } else if (IPSK == IPSK_bitmap) {
+ VarPrefix = getInstrProfBitmapVarPrefix();
+ VarName = getVarName(Inc, VarPrefix, Renamed);
+ InstrProfMCDCBitmapInstBase *BitmapUpdate =
+ dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
+ Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
+ } else {
+ llvm_unreachable("Profile Section must be for Counters or Bitmaps");
+ }
+
+ Ptr->setVisibility(Visibility);
+ // Put the counters and bitmaps in their own sections so linkers can
+ // remove unneeded sections.
+ Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));
+ Ptr->setLinkage(Linkage);
+ maybeSetComdat(Ptr, Fn, VarName);
+ return Ptr;
+}
+
+GlobalVariable *
+InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
+ StringRef Name,
+ GlobalValue::LinkageTypes Linkage) {
+ uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue();
+ auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes);
+ auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage,
+ Constant::getNullValue(BitmapTy), Name);
+ GV->setAlignment(Align(1));
+ return GV;
+}
+
+GlobalVariable *
+InstrProfiling::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
+ GlobalVariable *NamePtr = Inc->getName();
+ auto &PD = ProfileDataMap[NamePtr];
+ if (PD.RegionBitmaps)
+ return PD.RegionBitmaps;
+
+ // If RegionBitmaps doesn't already exist, create it by first setting up
+ // the corresponding profile section.
+ auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
+ PD.RegionBitmaps = BitmapPtr;
+ PD.NumBitmapBytes = Inc->getNumBitmapBytes()->getZExtValue();
+ return PD.RegionBitmaps;
+}
+GlobalVariable *
+InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
+ GlobalValue::LinkageTypes Linkage) {
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
- LLVMContext &Ctx = M->getContext();
+ auto &Ctx = M->getContext();
+ GlobalVariable *GV;
+ if (isa<InstrProfCoverInst>(Inc)) {
+ auto *CounterTy = Type::getInt8Ty(Ctx);
+ auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
+ // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
+ std::vector<Constant *> InitialValues(NumCounters,
+ Constant::getAllOnesValue(CounterTy));
+ GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
+ ConstantArray::get(CounterArrTy, InitialValues),
+ Name);
+ GV->setAlignment(Align(1));
+ } else {
+ auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
+ GV = new GlobalVariable(*M, CounterTy, false, Linkage,
+ Constant::getNullValue(CounterTy), Name);
+ GV->setAlignment(Align(8));
+ }
+ return GV;
+}
+
+GlobalVariable *
+InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
+ GlobalVariable *NamePtr = Inc->getName();
+ auto &PD = ProfileDataMap[NamePtr];
+ if (PD.RegionCounters)
+ return PD.RegionCounters;
- auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage);
- CounterPtr->setVisibility(Visibility);
- CounterPtr->setSection(
- getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
- CounterPtr->setLinkage(Linkage);
- MaybeSetComdat(CounterPtr);
+ // If RegionCounters doesn't already exist, create it by first setting up
+ // the corresponding profile section.
+ auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
PD.RegionCounters = CounterPtr;
+
if (DebugInfoCorrelate) {
+ LLVMContext &Ctx = M->getContext();
+ Function *Fn = Inc->getParent()->getParent();
if (auto *SP = Fn->getSubprogram()) {
DIBuilder DB(*M, true, SP->getUnit());
Metadata *FunctionNameAnnotation[] = {
@@ -1056,16 +1230,58 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
Annotations);
CounterPtr->addDebugInfo(DICounter);
DB.finalize();
- } else {
- std::string Msg = ("Missing debug info for function " + Fn->getName() +
- "; required for profile correlation.")
- .str();
- Ctx.diagnose(
- DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
}
+
+ // Mark the counter variable as used so that it isn't optimized out.
+ CompilerUsedVars.push_back(PD.RegionCounters);
}
- auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
+ // Create the data variable (if it doesn't already exist).
+ createDataVariable(Inc);
+
+ return PD.RegionCounters;
+}
+
+void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) {
+ // When debug information is correlated to profile data, a data variable
+ // is not needed.
+ if (DebugInfoCorrelate)
+ return;
+
+ GlobalVariable *NamePtr = Inc->getName();
+ auto &PD = ProfileDataMap[NamePtr];
+
+ // Return if data variable was already created.
+ if (PD.DataVar)
+ return;
+
+ LLVMContext &Ctx = M->getContext();
+
+ Function *Fn = Inc->getParent()->getParent();
+ GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
+ GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
+
+ // Due to the limitation of binder as of 2021/09/28, the duplicate weak
+ // symbols in the same csect won't be discarded. When there are duplicate weak
+ // symbols, we can NOT guarantee that the relocations get resolved to the
+ // intended weak symbol, so we can not ensure the correctness of the relative
+ // CounterPtr, so we have to use private linkage for counter and data symbols.
+ if (TT.isOSBinFormatXCOFF()) {
+ Linkage = GlobalValue::PrivateLinkage;
+ Visibility = GlobalValue::DefaultVisibility;
+ }
+
+ bool DataReferencedByCode = profDataReferencedByCode(*M);
+ bool NeedComdat = needsComdatForCounter(*Fn, *M);
+ bool Renamed;
+
+ // The Data Variable section is anchored to profile counters.
+ std::string CntsVarName =
+ getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
+ std::string DataVarName =
+ getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
+
+ auto *Int8PtrTy = PointerType::getUnqual(Ctx);
// Allocate statically the array of pointers to value profile nodes for
// the current function.
Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
@@ -1079,19 +1295,18 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
*M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
ValuesVar->setVisibility(Visibility);
+ setGlobalVariableLargeSection(TT, *ValuesVar);
ValuesVar->setSection(
getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
ValuesVar->setAlignment(Align(8));
- MaybeSetComdat(ValuesVar);
- ValuesPtrExpr =
- ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
+ maybeSetComdat(ValuesVar, Fn, CntsVarName);
+ ValuesPtrExpr = ValuesVar;
}
- if (DebugInfoCorrelate) {
- // Mark the counter variable as used so that it isn't optimized out.
- CompilerUsedVars.push_back(PD.RegionCounters);
- return PD.RegionCounters;
- }
+ uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+ auto *CounterPtr = PD.RegionCounters;
+
+ uint64_t NumBitmapBytes = PD.NumBitmapBytes;
// Create data variable.
auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
@@ -1134,6 +1349,16 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
ConstantExpr::getPtrToInt(Data, IntPtrTy));
+ // Bitmaps are relative to the same data variable as profile counters.
+ GlobalVariable *BitmapPtr = PD.RegionBitmaps;
+ Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
+
+ if (BitmapPtr != nullptr) {
+ RelativeBitmapPtr =
+ ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy),
+ ConstantExpr::getPtrToInt(Data, IntPtrTy));
+ }
+
Constant *DataVals[] = {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
#include "llvm/ProfileData/InstrProfData.inc"
@@ -1143,7 +1368,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
Data->setVisibility(Visibility);
Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
- MaybeSetComdat(Data);
+ maybeSetComdat(Data, Fn, CntsVarName);
PD.DataVar = Data;
@@ -1155,8 +1380,6 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
NamePtr->setLinkage(GlobalValue::PrivateLinkage);
// Collect the referenced names to be used by emitNameData.
ReferencedNames.push_back(NamePtr);
-
- return PD.RegionCounters;
}
void InstrProfiling::emitVNodes() {
@@ -1201,6 +1424,7 @@ void InstrProfiling::emitVNodes() {
auto *VNodesVar = new GlobalVariable(
*M, VNodesTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
+ setGlobalVariableLargeSection(TT, *VNodesVar);
VNodesVar->setSection(
getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
VNodesVar->setAlignment(M->getDataLayout().getABITypeAlign(VNodesTy));
@@ -1228,6 +1452,7 @@ void InstrProfiling::emitNameData() {
GlobalValue::PrivateLinkage, NamesVal,
getInstrProfNamesVarName());
NamesSize = CompressedNameStr.size();
+ setGlobalVariableLargeSection(TT, *NamesVar);
NamesVar->setSection(
getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
// On COFF, it's important to reduce the alignment down to 1 to prevent the
@@ -1248,7 +1473,7 @@ void InstrProfiling::emitRegistration() {
// Construct the function.
auto *VoidTy = Type::getVoidTy(M->getContext());
- auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
+ auto *VoidPtrTy = PointerType::getUnqual(M->getContext());
auto *Int64Ty = Type::getInt64Ty(M->getContext());
auto *RegisterFTy = FunctionType::get(VoidTy, false);
auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
@@ -1265,10 +1490,10 @@ void InstrProfiling::emitRegistration() {
IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
for (Value *Data : CompilerUsedVars)
if (!isa<Function>(Data))
- IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
+ IRB.CreateCall(RuntimeRegisterF, Data);
for (Value *Data : UsedVars)
if (Data != NamesVar && !isa<Function>(Data))
- IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
+ IRB.CreateCall(RuntimeRegisterF, Data);
if (NamesVar) {
Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
@@ -1277,8 +1502,7 @@ void InstrProfiling::emitRegistration() {
auto *NamesRegisterF =
Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
getInstrProfNamesRegFuncName(), M);
- IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
- IRB.getInt64(NamesSize)});
+ IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
}
IRB.CreateRetVoid();
generated by cgit v1.3 (git 2.53.0) at 2026-03-10 16:20:03 +0000