summaryrefslogtreecommitdiff
path: root/lib/Transforms/IPO/SampleProfile.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/IPO/SampleProfile.cpp')
-rw-r--r--lib/Transforms/IPO/SampleProfile.cpp131
1 files changed, 87 insertions, 44 deletions
diff --git a/lib/Transforms/IPO/SampleProfile.cpp b/lib/Transforms/IPO/SampleProfile.cpp
index dcd24595f7ea..9f123c2b875e 100644
--- a/lib/Transforms/IPO/SampleProfile.cpp
+++ b/lib/Transforms/IPO/SampleProfile.cpp
@@ -96,6 +96,13 @@ static cl::opt<std::string> SampleProfileFile(
"sample-profile-file", cl::init(""), cl::value_desc("filename"),
cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
+// The named file contains a set of transformations that may have been applied
+// to the symbol names between the program from which the sample data was
+// collected and the current program's symbols.
+static cl::opt<std::string> SampleProfileRemappingFile(
+ "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
+ cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
+
static cl::opt<unsigned> SampleProfileMaxPropagateIterations(
"sample-profile-max-propagate-iterations", cl::init(100),
cl::desc("Maximum number of iterations to go through when propagating "
@@ -116,6 +123,12 @@ static cl::opt<bool> NoWarnSampleUnused(
cl::desc("Use this option to turn off/on warnings about function with "
"samples but without debug information to use those samples. "));
+static cl::opt<bool> ProfileSampleAccurate(
+ "profile-sample-accurate", cl::Hidden, cl::init(false),
+ cl::desc("If the sample profile is accurate, we will mark all un-sampled "
+ "callsite and function as having 0 samples. Otherwise, treat "
+ "un-sampled callsites and functions conservatively as unknown. "));
+
namespace {
using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>;
@@ -183,12 +196,12 @@ private:
class SampleProfileLoader {
public:
SampleProfileLoader(
- StringRef Name, bool IsThinLTOPreLink,
+ StringRef Name, StringRef RemapName, bool IsThinLTOPreLink,
std::function<AssumptionCache &(Function &)> GetAssumptionCache,
std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo)
: GetAC(std::move(GetAssumptionCache)),
GetTTI(std::move(GetTargetTransformInfo)), Filename(Name),
- IsThinLTOPreLink(IsThinLTOPreLink) {}
+ RemappingFilename(RemapName), IsThinLTOPreLink(IsThinLTOPreLink) {}
bool doInitialization(Module &M);
bool runOnModule(Module &M, ModuleAnalysisManager *AM,
@@ -205,6 +218,7 @@ protected:
const FunctionSamples *findCalleeFunctionSamples(const Instruction &I) const;
std::vector<const FunctionSamples *>
findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
+ mutable DenseMap<const DILocation *, const FunctionSamples *> DILocation2SampleMap;
const FunctionSamples *findFunctionSamples(const Instruction &I) const;
bool inlineCallInstruction(Instruction *I);
bool inlineHotFunctions(Function &F,
@@ -282,6 +296,9 @@ protected:
/// Name of the profile file to load.
std::string Filename;
+ /// Name of the profile remapping file to load.
+ std::string RemappingFilename;
+
/// Flag indicating whether the profile input loaded successfully.
bool ProfileIsValid = false;
@@ -311,13 +328,14 @@ public:
SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile,
bool IsThinLTOPreLink = false)
- : ModulePass(ID), SampleLoader(Name, IsThinLTOPreLink,
- [&](Function &F) -> AssumptionCache & {
- return ACT->getAssumptionCache(F);
- },
- [&](Function &F) -> TargetTransformInfo & {
- return TTIWP->getTTI(F);
- }) {
+ : ModulePass(ID),
+ SampleLoader(Name, SampleProfileRemappingFile, IsThinLTOPreLink,
+ [&](Function &F) -> AssumptionCache & {
+ return ACT->getAssumptionCache(F);
+ },
+ [&](Function &F) -> TargetTransformInfo & {
+ return TTIWP->getTTI(F);
+ }) {
initializeSampleProfileLoaderLegacyPassPass(
*PassRegistry::getPassRegistry());
}
@@ -527,10 +545,10 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
if (!FS)
return std::error_code();
- // Ignore all intrinsics and branch instructions.
- // Branch instruction usually contains debug info from sources outside of
+ // Ignore all intrinsics, phinodes and branch instructions.
+ // Branch and phinodes instruction usually contains debug info from sources outside of
// the residing basic block, thus we ignore them during annotation.
- if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst))
+ if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst) || isa<PHINode>(Inst))
return std::error_code();
// If a direct call/invoke instruction is inlined in profile
@@ -643,8 +661,6 @@ SampleProfileLoader::findCalleeFunctionSamples(const Instruction &Inst) const {
if (FS == nullptr)
return nullptr;
- std::string CalleeGUID;
- CalleeName = getRepInFormat(CalleeName, Reader->getFormat(), CalleeGUID);
return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL),
DIL->getBaseDiscriminator()),
CalleeName);
@@ -683,10 +699,12 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
Sum += NameFS.second.getEntrySamples();
R.push_back(&NameFS.second);
}
- llvm::sort(R.begin(), R.end(),
- [](const FunctionSamples *L, const FunctionSamples *R) {
- return L->getEntrySamples() > R->getEntrySamples();
- });
+ llvm::sort(R, [](const FunctionSamples *L, const FunctionSamples *R) {
+ if (L->getEntrySamples() != R->getEntrySamples())
+ return L->getEntrySamples() > R->getEntrySamples();
+ return FunctionSamples::getGUID(L->getName()) <
+ FunctionSamples::getGUID(R->getName());
+ });
}
return R;
}
@@ -702,12 +720,14 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
/// \returns the FunctionSamples pointer to the inlined instance.
const FunctionSamples *
SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
- SmallVector<std::pair<LineLocation, StringRef>, 10> S;
const DILocation *DIL = Inst.getDebugLoc();
if (!DIL)
return Samples;
- return Samples->findFunctionSamples(DIL);
+ auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
+ if (it.second)
+ it.first->second = Samples->findFunctionSamples(DIL);
+ return it.first->second;
}
bool SampleProfileLoader::inlineCallInstruction(Instruction *I) {
@@ -760,7 +780,6 @@ bool SampleProfileLoader::inlineHotFunctions(
Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
DenseSet<Instruction *> PromotedInsns;
bool Changed = false;
- bool isCompact = (Reader->getFormat() == SPF_Compact_Binary);
while (true) {
bool LocalChanged = false;
SmallVector<Instruction *, 10> CIS;
@@ -792,19 +811,16 @@ bool SampleProfileLoader::inlineHotFunctions(
for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
if (IsThinLTOPreLink) {
FS->findInlinedFunctions(InlinedGUIDs, F.getParent(),
- PSI->getOrCompHotCountThreshold(),
- isCompact);
+ PSI->getOrCompHotCountThreshold());
continue;
}
- auto CalleeFunctionName = FS->getName();
+ auto CalleeFunctionName = FS->getFuncNameInModule(F.getParent());
// If it is a recursive call, we do not inline it as it could bloat
// the code exponentially. There is way to better handle this, e.g.
// clone the caller first, and inline the cloned caller if it is
// recursive. As llvm does not inline recursive calls, we will
// simply ignore it instead of handling it explicitly.
- std::string FGUID;
- auto Fname = getRepInFormat(F.getName(), Reader->getFormat(), FGUID);
- if (CalleeFunctionName == Fname)
+ if (CalleeFunctionName == F.getName())
continue;
const char *Reason = "Callee function not available";
@@ -834,8 +850,7 @@ bool SampleProfileLoader::inlineHotFunctions(
LocalChanged = true;
} else if (IsThinLTOPreLink) {
findCalleeFunctionSamples(*I)->findInlinedFunctions(
- InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold(),
- isCompact);
+ InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold());
}
}
if (LocalChanged) {
@@ -1177,14 +1192,13 @@ static SmallVector<InstrProfValueData, 2> SortCallTargets(
const SampleRecord::CallTargetMap &M) {
SmallVector<InstrProfValueData, 2> R;
for (auto I = M.begin(); I != M.end(); ++I)
- R.push_back({Function::getGUID(I->getKey()), I->getValue()});
- llvm::sort(R.begin(), R.end(),
- [](const InstrProfValueData &L, const InstrProfValueData &R) {
- if (L.Count == R.Count)
- return L.Value > R.Value;
- else
- return L.Count > R.Count;
- });
+ R.push_back({FunctionSamples::getGUID(I->getKey()), I->getValue()});
+ llvm::sort(R, [](const InstrProfValueData &L, const InstrProfValueData &R) {
+ if (L.Count == R.Count)
+ return L.Value > R.Value;
+ else
+ return L.Count > R.Count;
+ });
return R;
}
@@ -1292,7 +1306,7 @@ void SampleProfileLoader::propagateWeights(Function &F) {
}
}
}
- TerminatorInst *TI = BB->getTerminator();
+ Instruction *TI = BB->getTerminator();
if (TI->getNumSuccessors() == 1)
continue;
if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
@@ -1519,12 +1533,28 @@ bool SampleProfileLoader::doInitialization(Module &M) {
return false;
}
Reader = std::move(ReaderOrErr.get());
+ Reader->collectFuncsToUse(M);
ProfileIsValid = (Reader->read() == sampleprof_error::success);
+
+ if (!RemappingFilename.empty()) {
+ // Apply profile remappings to the loaded profile data if requested.
+ // For now, we only support remapping symbols encoded using the Itanium
+ // C++ ABI's name mangling scheme.
+ ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
+ RemappingFilename, Ctx, std::move(Reader));
+ if (std::error_code EC = ReaderOrErr.getError()) {
+ std::string Msg = "Could not open profile remapping file: " + EC.message();
+ Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
+ return false;
+ }
+ Reader = std::move(ReaderOrErr.get());
+ ProfileIsValid = (Reader->read() == sampleprof_error::success);
+ }
return true;
}
ModulePass *llvm::createSampleProfileLoaderPass() {
- return new SampleProfileLoaderLegacyPass(SampleProfileFile);
+ return new SampleProfileLoaderLegacyPass();
}
ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) {
@@ -1533,6 +1563,7 @@ ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) {
bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
ProfileSummaryInfo *_PSI) {
+ FunctionSamples::GUIDToFuncNameMapper Mapper(M);
if (!ProfileIsValid)
return false;
@@ -1577,15 +1608,25 @@ bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) {
ACT = &getAnalysis<AssumptionCacheTracker>();
TTIWP = &getAnalysis<TargetTransformInfoWrapperPass>();
ProfileSummaryInfo *PSI =
- getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
+ &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
return SampleLoader.runOnModule(M, nullptr, PSI);
}
bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) {
- // Initialize the entry count to -1, which will be treated conservatively
- // by getEntryCount as the same as unknown (None). If we have samples this
- // will be overwritten in emitAnnotations.
- F.setEntryCount(ProfileCount(-1, Function::PCT_Real));
+
+ DILocation2SampleMap.clear();
+ // By default the entry count is initialized to -1, which will be treated
+ // conservatively by getEntryCount as the same as unknown (None). This is
+ // to avoid newly added code to be treated as cold. If we have samples
+ // this will be overwritten in emitAnnotations.
+ // If ProfileSampleAccurate is true or F has profile-sample-accurate
+ // attribute, initialize the entry count to 0 so callsites or functions
+ // unsampled will be treated as cold.
+ uint64_t initialEntryCount =
+ (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate"))
+ ? 0
+ : -1;
+ F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real));
std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
if (AM) {
auto &FAM =
@@ -1616,6 +1657,8 @@ PreservedAnalyses SampleProfileLoaderPass::run(Module &M,
SampleProfileLoader SampleLoader(
ProfileFileName.empty() ? SampleProfileFile : ProfileFileName,
+ ProfileRemappingFileName.empty() ? SampleProfileRemappingFile
+ : ProfileRemappingFileName,
IsThinLTOPreLink, GetAssumptionCache, GetTTI);
SampleLoader.doInitialization(M);
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-07 12:28:46 +0000