diff options
Diffstat (limited to 'llvm/lib/ProfileData/SampleProfReader.cpp')
| -rw-r--r-- | llvm/lib/ProfileData/SampleProfReader.cpp | 297 |
1 files changed, 216 insertions, 81 deletions
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp index c42931174bc0..6058eddb13dc 100644 --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -26,6 +26,7 @@ #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Compression.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/LEB128.h" @@ -38,12 +39,22 @@ #include <cstdint> #include <limits> #include <memory> +#include <set> #include <system_error> #include <vector> using namespace llvm; using namespace sampleprof; +#define DEBUG_TYPE "samplepgo-reader" + +// This internal option specifies if the profile uses FS discriminators. +// It only applies to text, binary and compact binary format profiles. +// For ext-binary format profiles, the flag is set in the summary. +static cl::opt<bool> ProfileIsFSDisciminator( + "profile-isfs", cl::Hidden, cl::init(false), + cl::desc("Profile uses flow senstive discriminators")); + /// Dump the function profile for \p FName. /// /// \param FName Name of the function to print. @@ -87,13 +98,22 @@ static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } /// Possible metadata: /// - CFG Checksum information: /// !CFGChecksum: 12345 +/// - CFG Checksum information: +/// !Attributes: 1 /// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash. -static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash) { - if (!Input.startswith("!CFGChecksum:")) - return false; +static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash, + uint32_t &Attributes) { + if (Input.startswith("!CFGChecksum:")) { + StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim(); + return !CFGInfo.getAsInteger(10, FunctionHash); + } + + if (Input.startswith("!Attributes:")) { + StringRef Attrib = Input.substr(strlen("!Attributes:")).trim(); + return !Attrib.getAsInteger(10, Attributes); + } - StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim(); - return !CFGInfo.getAsInteger(10, FunctionHash); + return false; } enum class LineType { @@ -118,7 +138,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, DenseMap<StringRef, uint64_t> &TargetCountMap, - uint64_t &FunctionHash) { + uint64_t &FunctionHash, uint32_t &Attributes) { for (Depth = 0; Input[Depth] == ' '; Depth++) ; if (Depth == 0) @@ -126,7 +146,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, if (Depth == 1 && Input[Depth] == '!') { LineTy = LineType::Metadata; - return parseMetadata(Input.substr(Depth), FunctionHash); + return parseMetadata(Input.substr(Depth), FunctionHash, Attributes); } size_t n1 = Input.find(':'); @@ -222,14 +242,13 @@ std::error_code SampleProfileReaderText::readImpl() { sampleprof_error Result = sampleprof_error::success; InlineCallStack InlineStack; - int CSProfileCount = 0; - int RegularProfileCount = 0; uint32_t ProbeProfileCount = 0; // SeenMetadata tracks whether we have processed metadata for the current // top-level function profile. bool SeenMetadata = false; + ProfileIsFS = ProfileIsFSDisciminator; for (; !LineIt.is_at_eof(); ++LineIt) { if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') continue; @@ -257,11 +276,9 @@ std::error_code SampleProfileReaderText::readImpl() { SampleContext FContext(FName); if (FContext.hasContext()) ++CSProfileCount; - else - ++RegularProfileCount; Profiles[FContext] = FunctionSamples(); FunctionSamples &FProfile = Profiles[FContext]; - FProfile.setName(FContext.getName()); + FProfile.setName(FContext.getNameWithoutContext()); FProfile.setContext(FContext); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); @@ -273,9 +290,11 @@ std::error_code SampleProfileReaderText::readImpl() { DenseMap<StringRef, uint64_t> TargetCountMap; uint32_t Depth, LineOffset, Discriminator; LineType LineTy; - uint64_t FunctionHash; + uint64_t FunctionHash = 0; + uint32_t Attributes = 0; if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset, - Discriminator, FName, TargetCountMap, FunctionHash)) { + Discriminator, FName, TargetCountMap, FunctionHash, + Attributes)) { reportError(LineIt.line_number(), "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); @@ -287,6 +306,10 @@ std::error_code SampleProfileReaderText::readImpl() { "Found non-metadata after metadata: " + *LineIt); return sampleprof_error::malformed; } + + // Here we handle FS discriminators. + Discriminator &= getDiscriminatorMask(); + while (InlineStack.size() > Depth) { InlineStack.pop_back(); } @@ -315,8 +338,12 @@ std::error_code SampleProfileReaderText::readImpl() { } case LineType::Metadata: { FunctionSamples &FProfile = *InlineStack.back(); - FProfile.setFunctionHash(FunctionHash); - ++ProbeProfileCount; + if (FunctionHash) { + FProfile.setFunctionHash(FunctionHash); + ++ProbeProfileCount; + } + if (Attributes) + FProfile.getContext().setAllAttributes(Attributes); SeenMetadata = true; break; } @@ -324,13 +351,14 @@ std::error_code SampleProfileReaderText::readImpl() { } } - assert((RegularProfileCount == 0 || CSProfileCount == 0) && + assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile"); ProfileIsCS = (CSProfileCount > 0); assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles"); ProfileIsProbeBased = (ProbeProfileCount > 0); FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + FunctionSamples::ProfileIsCS = ProfileIsCS; if (Result == sampleprof_error::success) computeSummary(); @@ -491,6 +519,9 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { if (std::error_code EC = NumCalls.getError()) return EC; + // Here we handle FS discriminators: + uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); + for (uint32_t J = 0; J < *NumCalls; ++J) { auto CalledFunction(readStringFromTable()); if (std::error_code EC = CalledFunction.getError()) @@ -500,11 +531,11 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { if (std::error_code EC = CalledFunctionSamples.getError()) return EC; - FProfile.addCalledTargetSamples(*LineOffset, *Discriminator, + FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal, *CalledFunction, *CalledFunctionSamples); } - FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples); + FProfile.addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples); } // Read all the samples for inlined function calls. @@ -525,8 +556,11 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { if (std::error_code EC = FName.getError()) return EC; + // Here we handle FS discriminators: + uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); + FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( - LineLocation(*LineOffset, *Discriminator))[std::string(*FName)]; + LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)]; CalleeProfile.setName(*FName); if (std::error_code EC = readProfile(CalleeProfile)) return EC; @@ -546,18 +580,23 @@ SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { if (std::error_code EC = FName.getError()) return EC; - Profiles[*FName] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[*FName]; - FProfile.setName(*FName); - + SampleContext FContext(*FName); + Profiles[FContext] = FunctionSamples(); + FunctionSamples &FProfile = Profiles[FContext]; + FProfile.setName(FContext.getNameWithoutContext()); + FProfile.setContext(FContext); FProfile.addHeadSamples(*NumHeadSamples); + if (FContext.hasContext()) + CSProfileCount++; + if (std::error_code EC = readProfile(FProfile)) return EC; return sampleprof_error::success; } std::error_code SampleProfileReaderBinary::readImpl() { + ProfileIsFS = ProfileIsFSDisciminator; while (!at_eof()) { if (std::error_code EC = readFuncProfile(Data)) return EC; @@ -576,6 +615,10 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection( return EC; if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) Summary->setPartialProfile(true); + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) + FunctionSamples::ProfileIsCS = ProfileIsCS = true; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) + FunctionSamples::ProfileIsFS = ProfileIsFS = true; break; case SecNameTable: { FixedLengthMD5 = @@ -583,6 +626,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection( bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name); assert((!FixedLengthMD5 || UseMD5) && "If FixedLengthMD5 is true, UseMD5 has to be true"); + FunctionSamples::HasUniqSuffix = + hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix); if (std::error_code EC = readNameTableSec(UseMD5)) return EC; break; @@ -595,13 +640,16 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection( if (std::error_code EC = readFuncOffsetTable()) return EC; break; - case SecFuncMetadata: + case SecFuncMetadata: { ProfileIsProbeBased = hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; - if (std::error_code EC = readFuncMetadata()) + bool HasAttribute = + hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute); + if (std::error_code EC = readFuncMetadata(HasAttribute)) return EC; break; + } case SecProfileSymbolList: if (std::error_code EC = readProfileSymbolList()) return EC; @@ -614,11 +662,13 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection( return sampleprof_error::success; } -void SampleProfileReaderExtBinaryBase::collectFuncsFrom(const Module &M) { - UseAllFuncs = false; +bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() { + if (!M) + return false; FuncsToUse.clear(); - for (auto &F : M) + for (auto &F : *M) FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); + return true; } std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { @@ -647,47 +697,100 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { } std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { + // Collect functions used by current module if the Reader has been + // given a module. + // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName + // which will query FunctionSamples::HasUniqSuffix, so it has to be + // called after FunctionSamples::HasUniqSuffix is set, i.e. after + // NameTable section is read. + bool LoadFuncsToBeUsed = collectFuncsFromModule(); + + // When LoadFuncsToBeUsed is false, load all the function profiles. const uint8_t *Start = Data; - if (UseAllFuncs) { + if (!LoadFuncsToBeUsed) { while (Data < End) { if (std::error_code EC = readFuncProfile(Data)) return EC; } assert(Data == End && "More data is read than expected"); - return sampleprof_error::success; - } - - if (Remapper) { - for (auto Name : FuncsToUse) { - Remapper->insert(Name); + } else { + // Load function profiles on demand. + if (Remapper) { + for (auto Name : FuncsToUse) { + Remapper->insert(Name); + } } - } - if (useMD5()) { - for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) - continue; - const uint8_t *FuncProfileAddr = Start + iter->second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; - } - } else { - for (auto NameOffset : FuncOffsetTable) { - auto FuncName = NameOffset.first; - if (!FuncsToUse.count(FuncName) && - (!Remapper || !Remapper->exist(FuncName))) - continue; - const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; + if (useMD5()) { + for (auto Name : FuncsToUse) { + auto GUID = std::to_string(MD5Hash(Name)); + auto iter = FuncOffsetTable.find(StringRef(GUID)); + if (iter == FuncOffsetTable.end()) + continue; + const uint8_t *FuncProfileAddr = Start + iter->second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } + } else if (FunctionSamples::ProfileIsCS) { + // Compute the ordered set of names, so we can + // get all context profiles under a subtree by + // iterating through the ordered names. + struct Comparer { + // Ignore the closing ']' when ordering context + bool operator()(const StringRef &L, const StringRef &R) const { + return L.substr(0, L.size() - 1) < R.substr(0, R.size() - 1); + } + }; + std::set<StringRef, Comparer> OrderedNames; + for (auto Name : FuncOffsetTable) { + OrderedNames.insert(Name.first); + } + + // For each function in current module, load all + // context profiles for the function. + for (auto NameOffset : FuncOffsetTable) { + StringRef ContextName = NameOffset.first; + SampleContext FContext(ContextName); + auto FuncName = FContext.getNameWithoutContext(); + if (!FuncsToUse.count(FuncName) && + (!Remapper || !Remapper->exist(FuncName))) + continue; + + // For each context profile we need, try to load + // all context profile in the subtree. This can + // help profile guided importing for ThinLTO. + auto It = OrderedNames.find(ContextName); + while (It != OrderedNames.end() && + It->startswith(ContextName.substr(0, ContextName.size() - 1))) { + const uint8_t *FuncProfileAddr = Start + FuncOffsetTable[*It]; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + // Remove loaded context profile so we won't + // load it repeatedly. + It = OrderedNames.erase(It); + } + } + } else { + for (auto NameOffset : FuncOffsetTable) { + SampleContext FContext(NameOffset.first); + auto FuncName = FContext.getNameWithoutContext(); + if (!FuncsToUse.count(FuncName) && + (!Remapper || !Remapper->exist(FuncName))) + continue; + const uint8_t *FuncProfileAddr = Start + NameOffset.second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } } + Data = End; } - - Data = End; + assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && + "Cannot have both context-sensitive and regular profile"); + assert(ProfileIsCS == (CSProfileCount > 0) && + "Section flag should be consistent with actual profile"); return sampleprof_error::success; } @@ -778,13 +881,18 @@ std::error_code SampleProfileReaderExtBinaryBase::readImpl() { } std::error_code SampleProfileReaderCompactBinary::readImpl() { + // Collect functions used by current module if the Reader has been + // given a module. + bool LoadFuncsToBeUsed = collectFuncsFromModule(); + ProfileIsFS = ProfileIsFSDisciminator; std::vector<uint64_t> OffsetsToUse; - if (UseAllFuncs) { + if (!LoadFuncsToBeUsed) { + // load all the function profiles. for (auto FuncEntry : FuncOffsetTable) { OffsetsToUse.push_back(FuncEntry.second); } - } - else { + } else { + // load function profiles on demand. for (auto Name : FuncsToUse) { auto GUID = std::to_string(MD5Hash(Name)); auto iter = FuncOffsetTable.find(StringRef(GUID)); @@ -875,20 +983,34 @@ std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5) { return SampleProfileReaderBinary::readNameTable(); } -std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() { - if (!ProfileIsProbeBased) - return sampleprof_error::success; - for (unsigned I = 0; I < Profiles.size(); ++I) { +std::error_code +SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { + while (Data < End) { auto FName(readStringFromTable()); if (std::error_code EC = FName.getError()) return EC; - auto Checksum = readNumber<uint64_t>(); - if (std::error_code EC = Checksum.getError()) - return EC; + SampleContext FContext(*FName); + bool ProfileInMap = Profiles.count(FContext); - Profiles[*FName].setFunctionHash(*Checksum); + if (ProfileIsProbeBased) { + auto Checksum = readNumber<uint64_t>(); + if (std::error_code EC = Checksum.getError()) + return EC; + if (ProfileInMap) + Profiles[FContext].setFunctionHash(*Checksum); + } + + if (ProfileHasAttribute) { + auto Attributes = readNumber<uint32_t>(); + if (std::error_code EC = Attributes.getError()) + return EC; + if (ProfileInMap) + Profiles[FContext].getContext().setAllAttributes(*Attributes); + } } + + assert(Data == End && "More data is read than expected"); return sampleprof_error::success; } @@ -999,10 +1121,16 @@ static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) { Flags.append("fixlenmd5,"); else if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)) Flags.append("md5,"); + if (hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix)) + Flags.append("uniq,"); break; case SecProfSummary: if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) Flags.append("partial,"); + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) + Flags.append("context,"); + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) + Flags.append("fs-discriminator,"); break; default: break; @@ -1106,11 +1234,13 @@ std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { return sampleprof_error::success; } -void SampleProfileReaderCompactBinary::collectFuncsFrom(const Module &M) { - UseAllFuncs = false; +bool SampleProfileReaderCompactBinary::collectFuncsFromModule() { + if (!M) + return false; FuncsToUse.clear(); - for (auto &F : M) + for (auto &F : *M) FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); + return true; } std::error_code SampleProfileReaderBinary::readSummaryEntry( @@ -1417,6 +1547,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( /// This format is generated by the Linux Perf conversion tool at /// https://github.com/google/autofdo. std::error_code SampleProfileReaderGCC::readImpl() { + assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator"); // Read the string table. if (std::error_code EC = readNameTable()) return EC; @@ -1471,7 +1602,7 @@ SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) { /// \returns an error code indicating the status of the buffer. static ErrorOr<std::unique_ptr<MemoryBuffer>> setupMemoryBuffer(const Twine &Filename) { - auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename); + auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename, /*IsText=*/true); if (std::error_code EC = BufferOrErr.getError()) return EC; auto Buffer = std::move(BufferOrErr.get()); @@ -1489,16 +1620,19 @@ setupMemoryBuffer(const Twine &Filename) { /// /// \param C The LLVM context to use to emit diagnostics. /// +/// \param P The FSDiscriminatorPass. +/// /// \param RemapFilename The file used for profile remapping. /// /// \returns an error code indicating the status of the created reader. ErrorOr<std::unique_ptr<SampleProfileReader>> SampleProfileReader::create(const std::string Filename, LLVMContext &C, + FSDiscriminatorPass P, const std::string RemapFilename) { auto BufferOrError = setupMemoryBuffer(Filename); if (std::error_code EC = BufferOrError.getError()) return EC; - return create(BufferOrError.get(), C, RemapFilename); + return create(BufferOrError.get(), C, P, RemapFilename); } /// Create a sample profile remapper from the given input, to remap the @@ -1556,11 +1690,14 @@ SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B, /// /// \param C The LLVM context to use to emit diagnostics. /// +/// \param P The FSDiscriminatorPass. +/// /// \param RemapFilename The file used for profile remapping. /// /// \returns an error code indicating the status of the created reader. ErrorOr<std::unique_ptr<SampleProfileReader>> SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, + FSDiscriminatorPass P, const std::string RemapFilename) { std::unique_ptr<SampleProfileReader> Reader; if (SampleProfileReaderRawBinary::hasFormat(*B)) @@ -1592,6 +1729,8 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, return EC; } + Reader->setDiscriminatorMaskedBitFrom(P); + return std::move(Reader); } @@ -1599,9 +1738,5 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, // profile. Binary format has the profile summary in its header. void SampleProfileReader::computeSummary() { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); - for (const auto &I : Profiles) { - const FunctionSamples &Profile = I.second; - Builder.addRecord(Profile); - } - Summary = Builder.getSummary(); + Summary = Builder.computeSummaryForProfiles(Profiles); } |