summaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp')
-rw-r--r--llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp132
1 files changed, 88 insertions, 44 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 0d257bb6bd52..ad21fec269ec 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -446,13 +446,12 @@ bool InstrProfiling::lowerIntrinsics(Function *F) {
bool MadeChange = false;
PromotionCandidates.clear();
for (BasicBlock &BB : *F) {
- for (auto I = BB.begin(), E = BB.end(); I != E;) {
- auto Instr = I++;
- InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
+ for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
+ InstrProfIncrementInst *Inc = castToIncrementInst(&Instr);
if (Inc) {
lowerIncrement(Inc);
MadeChange = true;
- } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+ } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
lowerValueProfileInst(Ind);
MadeChange = true;
}
@@ -520,6 +519,14 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) {
}
}
+static bool needsRuntimeHookUnconditionally(const Triple &TT) {
+ // On Fuchsia, we only need runtime hook if any counters are present.
+ if (TT.isOSFuchsia())
+ return false;
+
+ return true;
+}
+
/// Check if the module contains uses of any profiling intrinsics.
static bool containsProfilingIntrinsics(Module &M) {
if (auto *F = M.getFunction(
@@ -548,8 +555,11 @@ bool InstrProfiling::run(
UsedVars.clear();
TT = Triple(M.getTargetTriple());
+ bool MadeChange = false;
+
// Emit the runtime hook even if no counters are present.
- bool MadeChange = emitRuntimeHook();
+ if (needsRuntimeHookUnconditionally(TT))
+ MadeChange = emitRuntimeHook();
// Improve compile time by avoiding linear scans when there is no work.
GlobalVariable *CoverageNamesVar =
@@ -588,6 +598,7 @@ bool InstrProfiling::run(
emitVNodes();
emitNameData();
+ emitRuntimeHook();
emitRegistration();
emitUses();
emitInitialization();
@@ -692,7 +703,6 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
LoadInst *LI = dyn_cast<LoadInst>(&I);
if (!LI) {
IRBuilder<> Builder(&I);
- Type *Int64Ty = Type::getInt64Ty(M->getContext());
GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
if (!Bias) {
// Compiler must define this variable when runtime counter relocation
@@ -747,14 +757,18 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
}
/// Get the name of a profiling variable for a particular function.
-static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
+static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix,
+ bool &Renamed) {
StringRef NamePrefix = getInstrProfNameVarPrefix();
StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
Function *F = Inc->getParent()->getParent();
Module *M = F->getParent();
if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
- !canRenameComdatFunc(*F))
+ !canRenameComdatFunc(*F)) {
+ Renamed = false;
return (Prefix + Name).str();
+ }
+ Renamed = true;
uint64_t FuncHash = Inc->getHash()->getZExtValue();
SmallVector<char, 24> HashPostfix;
if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
@@ -848,6 +862,15 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
+ // Due to the limitation of binder as of 2021/09/28, the duplicate weak
+ // symbols in the same csect won't be discarded. When there are duplicate weak
+ // symbols, we can NOT guarantee that the relocations get resolved to the
+ // intended weak symbol, so we can not ensure the correctness of the relative
+ // CounterPtr, so we have to use private linkage for counter and data symbols.
+ if (TT.isOSBinFormatXCOFF()) {
+ Linkage = GlobalValue::PrivateLinkage;
+ Visibility = GlobalValue::DefaultVisibility;
+ }
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that only one
// copy of counters of the COMDAT function will be emitted after linking. Keep
@@ -867,8 +890,11 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
// discarded.
bool DataReferencedByCode = profDataReferencedByCode(*M);
bool NeedComdat = needsComdatForCounter(*Fn, *M);
- std::string CntsVarName = getVarName(Inc, getInstrProfCountersVarPrefix());
- std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix());
+ bool Renamed;
+ std::string CntsVarName =
+ getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
+ std::string DataVarName =
+ getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
auto MaybeSetComdat = [&](GlobalVariable *GV) {
bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
if (UseComdat) {
@@ -909,7 +935,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
auto *ValuesVar = new GlobalVariable(
*M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
- getVarName(Inc, getInstrProfValuesVarPrefix()));
+ getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
ValuesVar->setVisibility(Visibility);
ValuesVar->setSection(
getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
@@ -920,6 +946,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
}
// Create data variable.
+ auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
auto *Int16Ty = Type::getInt16Ty(Ctx);
auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
Type *DataTypes[] = {
@@ -936,10 +963,6 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
- Constant *DataVals[] = {
-#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
-#include "llvm/ProfileData/InstrProfData.inc"
- };
// If the data variable is not referenced by code (if we don't emit
// @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
// data variable live under linker GC, the data variable can be private. This
@@ -947,14 +970,30 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
//
// On COFF, a comdat leader cannot be local so we require DataReferencedByCode
// to be false.
- if (NS == 0 && (TT.isOSBinFormatELF() ||
- (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
+ //
+ // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
+ // that other copies must have the same CFG and cannot have value profiling.
+ // If no hash suffix, other profd copies may be referenced by code.
+ if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
+ (TT.isOSBinFormatELF() ||
+ (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
Linkage = GlobalValue::PrivateLinkage;
Visibility = GlobalValue::DefaultVisibility;
}
auto *Data =
- new GlobalVariable(*M, DataTy, false, Linkage,
- ConstantStruct::get(DataTy, DataVals), DataVarName);
+ new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
+ // Reference the counter variable with a label difference (link-time
+ // constant).
+ auto *RelativeCounterPtr =
+ ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
+ ConstantExpr::getPtrToInt(Data, IntPtrTy));
+
+ Constant *DataVals[] = {
+#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
+#include "llvm/ProfileData/InstrProfData.inc"
+ };
+ Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
+
Data->setVisibility(Visibility);
Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
@@ -1035,7 +1074,7 @@ void InstrProfiling::emitNameData() {
std::string CompressedNameStr;
if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
DoInstrProfNameCompression)) {
- report_fatal_error(toString(std::move(E)), false);
+ report_fatal_error(Twine(toString(std::move(E))), false);
}
auto &Ctx = M->getContext();
@@ -1102,9 +1141,9 @@ void InstrProfiling::emitRegistration() {
}
bool InstrProfiling::emitRuntimeHook() {
- // We expect the linker to be invoked with -u<hook_var> flag for Linux or
- // Fuchsia, in which case there is no need to emit the user function.
- if (TT.isOSLinux() || TT.isOSFuchsia())
+ // We expect the linker to be invoked with -u<hook_var> flag for Linux
+ // in which case there is no need to emit the external variable.
+ if (TT.isOSLinux())
return false;
// If the module's provided its own runtime, we don't need to do anything.
@@ -1117,23 +1156,28 @@ bool InstrProfiling::emitRuntimeHook() {
new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
nullptr, getInstrProfRuntimeHookVarName());
- // Make a function that uses it.
- auto *User = Function::Create(FunctionType::get(Int32Ty, false),
- GlobalValue::LinkOnceODRLinkage,
- getInstrProfRuntimeHookVarUseFuncName(), M);
- User->addFnAttr(Attribute::NoInline);
- if (Options.NoRedZone)
- User->addFnAttr(Attribute::NoRedZone);
- User->setVisibility(GlobalValue::HiddenVisibility);
- if (TT.supportsCOMDAT())
- User->setComdat(M->getOrInsertComdat(User->getName()));
+ if (TT.isOSBinFormatELF()) {
+ // Mark the user variable as used so that it isn't stripped out.
+ CompilerUsedVars.push_back(Var);
+ } else {
+ // Make a function that uses it.
+ auto *User = Function::Create(FunctionType::get(Int32Ty, false),
+ GlobalValue::LinkOnceODRLinkage,
+ getInstrProfRuntimeHookVarUseFuncName(), M);
+ User->addFnAttr(Attribute::NoInline);
+ if (Options.NoRedZone)
+ User->addFnAttr(Attribute::NoRedZone);
+ User->setVisibility(GlobalValue::HiddenVisibility);
+ if (TT.supportsCOMDAT())
+ User->setComdat(M->getOrInsertComdat(User->getName()));
- IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
- auto *Load = IRB.CreateLoad(Int32Ty, Var);
- IRB.CreateRet(Load);
+ IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
+ auto *Load = IRB.CreateLoad(Int32Ty, Var);
+ IRB.CreateRet(Load);
- // Mark the user variable as used so that it isn't stripped out.
- CompilerUsedVars.push_back(User);
+ // Mark the function as used so that it isn't stripped out.
+ CompilerUsedVars.push_back(User);
+ }
return true;
}
@@ -1142,12 +1186,12 @@ void InstrProfiling::emitUses() {
// GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
// we conservatively retain all unconditionally in the compiler.
//
- // On ELF, the linker can guarantee the associated sections will be retained
- // or discarded as a unit, so llvm.compiler.used is sufficient. Similarly on
- // COFF, if prof data is not referenced by code we use one comdat and ensure
- // this GC property as well. Otherwise, we have to conservatively make all of
- // the sections retained by the linker.
- if (TT.isOSBinFormatELF() ||
+ // On ELF and Mach-O, the linker can guarantee the associated sections will be
+ // retained or discarded as a unit, so llvm.compiler.used is sufficient.
+ // Similarly on COFF, if prof data is not referenced by code we use one comdat
+ // and ensure this GC property as well. Otherwise, we have to conservatively
+ // make all of the sections retained by the linker.
+ if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
(TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M)))
appendToCompilerUsed(*M, CompilerUsedVars);
else
generated by cgit v1.3 (git 2.53.0) at 2026-05-14 06:50:22 +0000