diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp | 235 |
1 files changed, 148 insertions, 87 deletions
diff --git a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp index 535f87968104..78006aab1541 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -41,23 +41,10 @@ using namespace llvm; using namespace sampleprof; -std::error_code SampleProfileWriter::writeFuncProfiles( - const StringMap<FunctionSamples> &ProfileMap) { - // Sort the ProfileMap by total samples. - typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples; +std::error_code +SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) { std::vector<NameFunctionSamples> V; - for (const auto &I : ProfileMap) { - assert(I.getKey() == I.second.getNameWithContext() && - "Inconsistent profile map"); - V.push_back(std::make_pair(I.second.getNameWithContext(), &I.second)); - } - llvm::stable_sort( - V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) { - if (A.second->getTotalSamples() == B.second->getTotalSamples()) - return A.first > B.first; - return A.second->getTotalSamples() > B.second->getTotalSamples(); - }); - + sortFuncProfiles(ProfileMap, V); for (const auto &I : V) { if (std::error_code EC = writeSample(*I.second)) return EC; @@ -65,8 +52,7 @@ std::error_code SampleProfileWriter::writeFuncProfiles( return sampleprof_error::success; } -std::error_code -SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) { +std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; @@ -130,8 +116,8 @@ std::error_code SampleProfileWriterExtBinaryBase::addNewSection( return sampleprof_error::success; } -std::error_code SampleProfileWriterExtBinaryBase::write( - const StringMap<FunctionSamples> &ProfileMap) { +std::error_code +SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; @@ -146,11 +132,28 @@ std::error_code SampleProfileWriterExtBinaryBase::write( return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx( + const SampleContext &Context) { + if (Context.hasContext()) + return writeCSNameIdx(Context); + else + return SampleProfileWriterBinary::writeNameIdx(Context.getName()); +} + +std::error_code +SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) { + const auto &Ret = CSNameTable.find(Context); + if (Ret == CSNameTable.end()) + return sampleprof_error::truncated_name_table; + encodeULEB128(Ret->second, *OutputStream); + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { uint64_t Offset = OutputStream->tell(); - StringRef Name = S.getNameWithContext(); - FuncOffsetTable[Name] = Offset - SecLBRProfileStart; + auto &Context = S.getContext(); + FuncOffsetTable[Context] = Offset - SecLBRProfileStart; encodeULEB128(S.getHeadSamples(), *OutputStream); return writeBody(S); } @@ -162,24 +165,42 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() { encodeULEB128(FuncOffsetTable.size(), OS); // Write out FuncOffsetTable. - for (auto Entry : FuncOffsetTable) { - if (std::error_code EC = - writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS)) + auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) { + if (std::error_code EC = writeContextIdx(Context)) return EC; - encodeULEB128(Entry.second, OS); + encodeULEB128(Offset, OS); + return (std::error_code)sampleprof_error::success; + }; + + if (FunctionSamples::ProfileIsCS) { + // Sort the contexts before writing them out. This is to help fast load all + // context profiles for a function as well as their callee contexts which + // can help profile-guided importing for ThinLTO. + std::map<SampleContext, uint64_t> OrderedFuncOffsetTable( + FuncOffsetTable.begin(), FuncOffsetTable.end()); + for (const auto &Entry : OrderedFuncOffsetTable) { + if (std::error_code EC = WriteItem(Entry.first, Entry.second)) + return EC; + } + addSectionFlag(SecFuncOffsetTable, SecFuncOffsetFlags::SecFlagOrdered); + } else { + for (const auto &Entry : FuncOffsetTable) { + if (std::error_code EC = WriteItem(Entry.first, Entry.second)) + return EC; + } } + FuncOffsetTable.clear(); return sampleprof_error::success; } std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( - const StringMap<FunctionSamples> &Profiles) { + const SampleProfileMap &Profiles) { if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS) return sampleprof_error::success; auto &OS = *OutputStream; for (const auto &Entry : Profiles) { - if (std::error_code EC = writeNameIdx(Entry.second.getNameWithContext(), - FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeContextIdx(Entry.second.getContext())) return EC; if (FunctionSamples::ProfileIsProbeBased) encodeULEB128(Entry.second.getFunctionHash(), OS); @@ -195,7 +216,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { auto &OS = *OutputStream; std::set<StringRef> V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the MD5 name table. We wrote unencoded MD5 so reader can // retrieve the name using the name index without having to read the @@ -208,11 +229,10 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { } std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( - const StringMap<FunctionSamples> &ProfileMap) { + const SampleProfileMap &ProfileMap) { for (const auto &I : ProfileMap) { - assert(I.first() == I.second.getNameWithContext() && - "Inconsistent profile map"); - addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS); + assert(I.first == I.second.getContext() && "Inconsistent profile map"); + addContext(I.second.getContext()); addNames(I.second); } @@ -220,7 +240,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( // so compiler won't strip the suffix during profile matching after // seeing the flag in the profile. for (const auto &I : NameTable) { - if (I.first.find(FunctionSamples::UniqSuffix) != StringRef::npos) { + if (I.first.contains(FunctionSamples::UniqSuffix)) { addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix); break; } @@ -231,6 +251,34 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() { + // Sort the names to make CSNameTable deterministic. + std::set<SampleContext> OrderedContexts; + for (const auto &I : CSNameTable) + OrderedContexts.insert(I.first); + assert(OrderedContexts.size() == CSNameTable.size() && + "Unmatched ordered and unordered contexts"); + uint64_t I = 0; + for (auto &Context : OrderedContexts) + CSNameTable[Context] = I++; + + auto &OS = *OutputStream; + encodeULEB128(OrderedContexts.size(), OS); + support::endian::Writer Writer(OS, support::little); + for (auto Context : OrderedContexts) { + auto Frames = Context.getContextFrames(); + encodeULEB128(Frames.size(), OS); + for (auto &Callsite : Frames) { + if (std::error_code EC = writeNameIdx(Callsite.FuncName)) + return EC; + encodeULEB128(Callsite.Location.LineOffset, OS); + encodeULEB128(Callsite.Location.Discriminator, OS); + } + } + + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() { if (ProfSymList && ProfSymList->size() > 0) @@ -241,8 +289,7 @@ SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() { } std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( - SecType Type, uint32_t LayoutIdx, - const StringMap<FunctionSamples> &ProfileMap) { + SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) { // The setting of SecFlagCompress should happen before markSectionStart. if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress()) setToCompressSection(SecProfileSymbolList); @@ -266,6 +313,10 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( if (auto EC = writeNameTableSection(ProfileMap)) return EC; break; + case SecCSNameTable: + if (auto EC = writeCSNameTableSection()) + return EC; + break; case SecLBRProfile: SecLBRProfileStart = OutputStream->tell(); if (std::error_code EC = writeFuncProfiles(ProfileMap)) @@ -294,7 +345,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( } std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( - const StringMap<FunctionSamples> &ProfileMap) { + const SampleProfileMap &ProfileMap) { // The const indices passed to writeOneSection below are specifying the // positions of the sections in SectionHdrLayout. Look at // initSectionHdrLayout to find out where each section is located in @@ -303,32 +354,33 @@ std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( return EC; if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecLBRProfile, 3, ProfileMap)) + if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecProfileSymbolList, 4, ProfileMap)) + if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ProfileMap)) + if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncMetadata, 5, ProfileMap)) + if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap)) return EC; return sampleprof_error::success; } -static void -splitProfileMapToTwo(const StringMap<FunctionSamples> &ProfileMap, - StringMap<FunctionSamples> &ContextProfileMap, - StringMap<FunctionSamples> &NoContextProfileMap) { +static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, + SampleProfileMap &ContextProfileMap, + SampleProfileMap &NoContextProfileMap) { for (const auto &I : ProfileMap) { if (I.second.getCallsiteSamples().size()) - ContextProfileMap.insert({I.first(), I.second}); + ContextProfileMap.insert({I.first, I.second}); else - NoContextProfileMap.insert({I.first(), I.second}); + NoContextProfileMap.insert({I.first, I.second}); } } std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout( - const StringMap<FunctionSamples> &ProfileMap) { - StringMap<FunctionSamples> ContextProfileMap, NoContextProfileMap; + const SampleProfileMap &ProfileMap) { + SampleProfileMap ContextProfileMap, NoContextProfileMap; splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap); if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap)) @@ -358,7 +410,7 @@ std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout( } std::error_code SampleProfileWriterExtBinary::writeSections( - const StringMap<FunctionSamples> &ProfileMap) { + const SampleProfileMap &ProfileMap) { std::error_code EC; if (SecLayout == DefaultLayout) EC = writeDefaultLayout(ProfileMap); @@ -369,8 +421,8 @@ std::error_code SampleProfileWriterExtBinary::writeSections( return EC; } -std::error_code SampleProfileWriterCompactBinary::write( - const StringMap<FunctionSamples> &ProfileMap) { +std::error_code +SampleProfileWriterCompactBinary::write(const SampleProfileMap &ProfileMap) { if (std::error_code EC = SampleProfileWriter::write(ProfileMap)) return EC; if (std::error_code EC = writeFuncOffsetTable()) @@ -389,7 +441,7 @@ std::error_code SampleProfileWriterCompactBinary::write( std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { auto &OS = *OutputStream; if (FunctionSamples::ProfileIsCS) - OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples(); + OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples(); else OS << S.getName() << ":" << S.getTotalSamples(); @@ -445,27 +497,28 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { return sampleprof_error::success; } -std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName, - bool IsContextName) { - std::string BracketedName; - if (IsContextName) { - BracketedName = "[" + FName.str() + "]"; - FName = StringRef(BracketedName); - } +std::error_code +SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) { + assert(!Context.hasContext() && "cs profile is not supported"); + return writeNameIdx(Context.getName()); +} - const auto &Ret = NameTable.find(FName); - if (Ret == NameTable.end()) +std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) { + auto &NTable = getNameTable(); + const auto &Ret = NTable.find(FName); + if (Ret == NTable.end()) return sampleprof_error::truncated_name_table; encodeULEB128(Ret->second, *OutputStream); return sampleprof_error::success; } -void SampleProfileWriterBinary::addName(StringRef FName, bool IsContextName) { - if (IsContextName) { - auto It = BracketedContextStr.insert("[" + FName.str() + "]"); - FName = StringRef(*It.first); - } - NameTable.insert(std::make_pair(FName, 0)); +void SampleProfileWriterBinary::addName(StringRef FName) { + auto &NTable = getNameTable(); + NTable.insert(std::make_pair(FName, 0)); +} + +void SampleProfileWriterBinary::addContext(const SampleContext &Context) { + addName(Context.getName()); } void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { @@ -485,7 +538,19 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { } } -void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) { +void SampleProfileWriterExtBinaryBase::addContext( + const SampleContext &Context) { + if (Context.hasContext()) { + for (auto &Callsite : Context.getContextFrames()) + SampleProfileWriterBinary::addName(Callsite.FuncName); + CSNameTable.insert(std::make_pair(Context, 0)); + } else { + SampleProfileWriterBinary::addName(Context.getName()); + } +} + +void SampleProfileWriterBinary::stablizeNameTable( + MapVector<StringRef, uint32_t> &NameTable, std::set<StringRef> &V) { // Sort the names to make NameTable deterministic. for (const auto &I : NameTable) V.insert(I.first); @@ -497,7 +562,7 @@ void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) { std::error_code SampleProfileWriterBinary::writeNameTable() { auto &OS = *OutputStream; std::set<StringRef> V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the name table. encodeULEB128(NameTable.size(), OS); @@ -526,8 +591,7 @@ std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() { // Write out FuncOffsetTable. for (auto Entry : FuncOffsetTable) { - if (std::error_code EC = - writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeNameIdx(Entry.first)) return EC; encodeULEB128(Entry.second, OS); } @@ -537,7 +601,7 @@ std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() { std::error_code SampleProfileWriterCompactBinary::writeNameTable() { auto &OS = *OutputStream; std::set<StringRef> V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the name table. encodeULEB128(NameTable.size(), OS); @@ -556,8 +620,8 @@ SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) { return sampleprof_error::success; } -std::error_code SampleProfileWriterBinary::writeHeader( - const StringMap<FunctionSamples> &ProfileMap) { +std::error_code +SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) { writeMagicIdent(Format); computeSummary(ProfileMap); @@ -566,9 +630,8 @@ std::error_code SampleProfileWriterBinary::writeHeader( // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { - assert(I.first() == I.second.getNameWithContext() && - "Inconsistent profile map"); - addName(I.first(), FunctionSamples::ProfileIsCS); + assert(I.first == I.second.getContext() && "Inconsistent profile map"); + addContext(I.first); addNames(I.second); } @@ -642,7 +705,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() { } std::error_code SampleProfileWriterExtBinaryBase::writeHeader( - const StringMap<FunctionSamples> &ProfileMap) { + const SampleProfileMap &ProfileMap) { auto &OS = *OutputStream; FileStart = OS.tell(); writeMagicIdent(Format); @@ -652,7 +715,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader( } std::error_code SampleProfileWriterCompactBinary::writeHeader( - const StringMap<FunctionSamples> &ProfileMap) { + const SampleProfileMap &ProfileMap) { support::endian::Writer Writer(*OutputStream, support::little); if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap)) return EC; @@ -671,7 +734,8 @@ std::error_code SampleProfileWriterBinary::writeSummary() { encodeULEB128(Summary->getMaxFunctionCount(), OS); encodeULEB128(Summary->getNumCounts(), OS); encodeULEB128(Summary->getNumFunctions(), OS); - std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary(); + const std::vector<ProfileSummaryEntry> &Entries = + Summary->getDetailedSummary(); encodeULEB128(Entries.size(), OS); for (auto Entry : Entries) { encodeULEB128(Entry.Cutoff, OS); @@ -682,9 +746,7 @@ std::error_code SampleProfileWriterBinary::writeSummary() { } std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { auto &OS = *OutputStream; - - if (std::error_code EC = - writeNameIdx(S.getNameWithContext(), FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeContextIdx(S.getContext())) return EC; encodeULEB128(S.getTotalSamples(), OS); @@ -803,8 +865,7 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, return std::move(Writer); } -void SampleProfileWriter::computeSummary( - const StringMap<FunctionSamples> &ProfileMap) { +void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); Summary = Builder.computeSummaryForProfiles(ProfileMap); } |