diff options
Diffstat (limited to 'include/llvm/ProfileData')
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMapping.h | 16 | ||||
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMappingWriter.h | 3 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProf.h | 18 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfReader.h | 12 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProf.h | 178 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProfReader.h | 272 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProfWriter.h | 118 |
7 files changed, 482 insertions, 135 deletions
diff --git a/include/llvm/ProfileData/Coverage/CoverageMapping.h b/include/llvm/ProfileData/Coverage/CoverageMapping.h index 11758ac4cf2f..0dd0c7ec8065 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -301,7 +301,12 @@ public: struct FunctionRecord { /// Raw function name. std::string Name; - /// Associated files. + /// Mapping from FileID (i.e. vector index) to filename. Used to support + /// macro expansions within a function in which the macro and function are + /// defined in separate files. + /// + /// TODO: Uniquing filenames across all function records may be a performance + /// optimization. std::vector<std::string> Filenames; /// Regions in the function along with their counts. std::vector<CountedRegion> CountedRegions; @@ -508,6 +513,7 @@ public: class CoverageMapping { DenseMap<size_t, DenseSet<size_t>> RecordProvenance; std::vector<FunctionRecord> Functions; + DenseMap<size_t, SmallVector<unsigned, 0>> FilenameHash2RecordIndices; std::vector<std::pair<std::string, uint64_t>> FuncHashMismatches; CoverageMapping() = default; @@ -516,6 +522,13 @@ class CoverageMapping { Error loadFunctionRecord(const CoverageMappingRecord &Record, IndexedInstrProfReader &ProfileReader); + /// Look up the indices for function records which are at least partially + /// defined in the specified file. This is guaranteed to return a superset of + /// such records: extra records not in the file may be included if there is + /// a hash collision on the filename. Clients must be robust to collisions. + ArrayRef<unsigned> + getImpreciseRecordIndicesForFilename(StringRef Filename) const; + public: CoverageMapping(const CoverageMapping &) = delete; CoverageMapping &operator=(const CoverageMapping &) = delete; @@ -527,6 +540,7 @@ public: /// Load the coverage mapping from the given object files and profile. If /// \p Arches is non-empty, it must specify an architecture for each object. + /// Ignores non-instrumented object files unless all are not instrumented. static Expected<std::unique_ptr<CoverageMapping>> load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename, ArrayRef<StringRef> Arches = None); diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h index 5f88cacdfcbb..6fcd8a09a494 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h @@ -30,8 +30,7 @@ class CoverageFilenamesSectionWriter { ArrayRef<StringRef> Filenames; public: - CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames) - : Filenames(Filenames) {} + CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames); /// Write encoded filenames to the given output stream. void write(raw_ostream &OS); diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index c7d764ade30d..c26f76949992 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -93,10 +93,6 @@ inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } /// Return the name of value profile node array variables: inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } -/// Return the name prefix of the COMDAT group for instrumentation variables -/// associated with a COMDAT function. -inline StringRef getInstrProfComdatPrefix() { return "__profv_"; } - /// Return the name of the variable holding the strings (possibly compressed) /// of all function's PGO names. inline StringRef getInstrProfNamesVarName() { @@ -634,8 +630,8 @@ struct OverlapStats { FuncHash = Hash; } - Error accumuateCounts(const std::string &BaseFilename, - const std::string &TestFilename, bool IsCS); + Error accumulateCounts(const std::string &BaseFilename, + const std::string &TestFilename, bool IsCS); void addOneMismatch(const CountSumOrPercent &MismatchFunc); void addOneUnique(const CountSumOrPercent &UniqueFunc); @@ -695,7 +691,7 @@ struct InstrProfRecord { InstrProfRecord(const InstrProfRecord &RHS) : Counts(RHS.Counts), ValueData(RHS.ValueData - ? llvm::make_unique<ValueProfData>(*RHS.ValueData) + ? std::make_unique<ValueProfData>(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { @@ -705,7 +701,7 @@ struct InstrProfRecord { return *this; } if (!ValueData) - ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData); + ValueData = std::make_unique<ValueProfData>(*RHS.ValueData); else *ValueData = *RHS.ValueData; return *this; @@ -772,7 +768,7 @@ struct InstrProfRecord { void clearValueData() { ValueData = nullptr; } /// Compute the sums of all counts and store in Sum. - void accumuateCounts(CountSumOrPercent &Sum) const; + void accumulateCounts(CountSumOrPercent &Sum) const; /// Compute the overlap b/w this IntrprofRecord and Other. void overlap(InstrProfRecord &Other, OverlapStats &Overlap, @@ -817,7 +813,7 @@ private: std::vector<InstrProfValueSiteRecord> & getOrCreateValueSitesForKind(uint32_t ValueKind) { if (!ValueData) - ValueData = llvm::make_unique<ValueProfData>(); + ValueData = std::make_unique<ValueProfData>(); switch (ValueKind) { case IPVK_IndirectCallTarget: return ValueData->IndirectCallSites; @@ -897,7 +893,7 @@ InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, return std::unique_ptr<InstrProfValueData[]>(nullptr); } - auto VD = llvm::make_unique<InstrProfValueData[]>(N); + auto VD = std::make_unique<InstrProfValueData[]>(N); TotalCount = getValueForSite(VD.get(), ValueKind, Site); return VD; diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h index 73751faab88e..f5f552672bf0 100644 --- a/include/llvm/ProfileData/InstrProfReader.h +++ b/include/llvm/ProfileData/InstrProfReader.h @@ -92,7 +92,7 @@ public: virtual InstrProfSymtab &getSymtab() = 0; /// Compute the sum of counts and return in Sum. - void accumuateCounts(CountSumOrPercent &Sum, bool IsCS); + void accumulateCounts(CountSumOrPercent &Sum, bool IsCS); protected: std::unique_ptr<InstrProfSymtab> Symtab; @@ -268,8 +268,14 @@ private: return (const char *)ValueDataStart; } - const uint64_t *getCounter(IntPtrT CounterPtr) const { - ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); + /// Get the offset of \p CounterPtr from the start of the counters section of + /// the profile. The offset has units of "number of counters", i.e. increasing + /// the offset by 1 corresponds to an increase in the *byte offset* by 8. + ptrdiff_t getCounterOffset(IntPtrT CounterPtr) const { + return (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); + } + + const uint64_t *getCounter(ptrdiff_t Offset) const { return CountersStart + Offset; } diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 7fbc857b7230..55418d9d0f9c 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -18,15 +18,18 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cstdint> #include <map> +#include <set> #include <string> #include <system_error> #include <utility> @@ -49,7 +52,10 @@ enum class sampleprof_error { truncated_name_table, not_implemented, counter_overflow, - ostream_seek_unsupported + ostream_seek_unsupported, + compress_failed, + uncompress_failed, + zlib_unavailable }; inline std::error_code make_error_code(sampleprof_error E) { @@ -83,6 +89,7 @@ enum SampleProfileFormat { SPF_Text = 0x1, SPF_Compact_Binary = 0x2, SPF_GCC = 0x3, + SPF_Ext_Binary = 0x4, SPF_Binary = 0xff }; @@ -105,6 +112,61 @@ static inline StringRef getRepInFormat(StringRef Name, static inline uint64_t SPVersion() { return 103; } +// Section Type used by SampleProfileExtBinaryBaseReader and +// SampleProfileExtBinaryBaseWriter. Never change the existing +// value of enum. Only append new ones. +enum SecType { + SecInValid = 0, + SecProfSummary = 1, + SecNameTable = 2, + SecProfileSymbolList = 3, + SecFuncOffsetTable = 4, + // marker for the first type of profile. + SecFuncProfileFirst = 32, + SecLBRProfile = SecFuncProfileFirst +}; + +static inline std::string getSecName(SecType Type) { + switch (Type) { + case SecInValid: + return "InvalidSection"; + case SecProfSummary: + return "ProfileSummarySection"; + case SecNameTable: + return "NameTableSection"; + case SecProfileSymbolList: + return "ProfileSymbolListSection"; + case SecFuncOffsetTable: + return "FuncOffsetTableSection"; + case SecLBRProfile: + return "LBRProfileSection"; + } + llvm_unreachable("A SecType has no name for output"); +} + +// Entry type of section header table used by SampleProfileExtBinaryBaseReader +// and SampleProfileExtBinaryBaseWriter. +struct SecHdrTableEntry { + SecType Type; + uint64_t Flags; + uint64_t Offset; + uint64_t Size; +}; + +enum SecFlags { SecFlagInValid = 0, SecFlagCompress = (1 << 0) }; + +static inline void addSecFlags(SecHdrTableEntry &Entry, uint64_t Flags) { + Entry.Flags |= Flags; +} + +static inline void removeSecFlags(SecHdrTableEntry &Entry, uint64_t Flags) { + Entry.Flags &= ~Flags; +} + +static inline bool hasSecFlag(SecHdrTableEntry &Entry, SecFlags Flag) { + return Entry.Flags & Flag; +} + /// Represents the relative location of an instruction. /// /// Instruction locations are specified by the line offset from the @@ -143,8 +205,18 @@ raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); /// will be a list of one or more functions. class SampleRecord { public: - using CallTargetMap = StringMap<uint64_t>; + using CallTarget = std::pair<StringRef, uint64_t>; + struct CallTargetComparator { + bool operator()(const CallTarget &LHS, const CallTarget &RHS) const { + if (LHS.second != RHS.second) + return LHS.second > RHS.second; + + return LHS.first < RHS.first; + } + }; + using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>; + using CallTargetMap = StringMap<uint64_t>; SampleRecord() = default; /// Increment the number of samples for this record by \p S. @@ -179,6 +251,18 @@ public: uint64_t getSamples() const { return NumSamples; } const CallTargetMap &getCallTargets() const { return CallTargets; } + const SortedCallTargetSet getSortedCallTargets() const { + return SortCallTargets(CallTargets); + } + + /// Sort call targets in descending order of call frequency. + static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) { + SortedCallTargetSet SortedTargets; + for (const auto &I : Targets) { + SortedTargets.emplace(I.first(), I.second); + } + return SortedTargets; + } /// Merge the samples in \p Other into this record. /// Optionally scale sample counts by \p Weight. @@ -205,7 +289,7 @@ class FunctionSamples; using BodySampleMap = std::map<LineLocation, SampleRecord>; // NOTE: Using a StringMap here makes parsed profiles consume around 17% more // memory, which is *very* significant for large profiles. -using FunctionSamplesMap = std::map<std::string, FunctionSamples>; +using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>; using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>; /// Representation of the samples collected for a function. @@ -447,11 +531,10 @@ public: StringRef getNameInModule(StringRef Name, const Module *M) const { if (Format != SPF_Compact_Binary) return Name; - // Expect CurrentModule to be initialized by GUIDToFuncNameMapper. - if (M != CurrentModule) - llvm_unreachable("Input Module should be the same as CurrentModule"); - auto iter = GUIDToFuncNameMap.find(std::stoull(Name.data())); - if (iter == GUIDToFuncNameMap.end()) + + assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first"); + auto iter = GUIDToFuncNameMap->find(std::stoull(Name.data())); + if (iter == GUIDToFuncNameMap->end()) return StringRef(); return iter->second; } @@ -472,42 +555,10 @@ public: const FunctionSamples *findFunctionSamples(const DILocation *DIL) const; static SampleProfileFormat Format; - /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for - /// all the function symbols defined or declared in CurrentModule. - static DenseMap<uint64_t, StringRef> GUIDToFuncNameMap; - static Module *CurrentModule; - - class GUIDToFuncNameMapper { - public: - GUIDToFuncNameMapper(Module &M) { - if (Format != SPF_Compact_Binary) - return; - - for (const auto &F : M) { - StringRef OrigName = F.getName(); - GUIDToFuncNameMap.insert({Function::getGUID(OrigName), OrigName}); - /// Local to global var promotion used by optimization like thinlto - /// will rename the var and add suffix like ".llvm.xxx" to the - /// original local name. In sample profile, the suffixes of function - /// names are all stripped. Since it is possible that the mapper is - /// built in post-thin-link phase and var promotion has been done, - /// we need to add the substring of function name without the suffix - /// into the GUIDToFuncNameMap. - StringRef CanonName = getCanonicalFnName(F); - if (CanonName != OrigName) - GUIDToFuncNameMap.insert({Function::getGUID(CanonName), CanonName}); - } - CurrentModule = &M; - } - - ~GUIDToFuncNameMapper() { - if (Format != SPF_Compact_Binary) - return; - GUIDToFuncNameMap.clear(); - CurrentModule = nullptr; - } - }; + /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for + /// all the function symbols defined or declared in current module. + DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr; // Assume the input \p Name is a name coming from FunctionSamples itself. // If the format is SPF_Compact_Binary, the name is already a GUID and we @@ -583,6 +634,47 @@ private: SamplesWithLocList V; }; +/// ProfileSymbolList records the list of function symbols shown up +/// in the binary used to generate the profile. It is useful to +/// to discriminate a function being so cold as not to shown up +/// in the profile and a function newly added. +class ProfileSymbolList { +public: + /// copy indicates whether we need to copy the underlying memory + /// for the input Name. + void add(StringRef Name, bool copy = false) { + if (!copy) { + Syms.insert(Name); + return; + } + Syms.insert(Name.copy(Allocator)); + } + + bool contains(StringRef Name) { return Syms.count(Name); } + + void merge(const ProfileSymbolList &List) { + for (auto Sym : List.Syms) + add(Sym, true); + } + + unsigned size() { return Syms.size(); } + + void setToCompress(bool TC) { ToCompress = TC; } + bool toCompress() { return ToCompress; } + + std::error_code read(const uint8_t *Data, uint64_t ListSize); + std::error_code write(raw_ostream &OS); + void dump(raw_ostream &OS = dbgs()) const; + +private: + // Determine whether or not to compress the symbol list when + // writing it into profile. The variable is unused when the symbol + // list is read from an existing profile. + bool ToCompress = false; + DenseSet<StringRef> Syms; + BumpPtrAllocator Allocator; +}; + } // end namespace sampleprof } // end namespace llvm diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index 969cdea859c9..5a5d4cfde224 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -235,6 +235,62 @@ class raw_ostream; namespace sampleprof { +class SampleProfileReader; + +/// SampleProfileReaderItaniumRemapper remaps the profile data from a +/// sample profile data reader, by applying a provided set of equivalences +/// between components of the symbol names in the profile. +class SampleProfileReaderItaniumRemapper { +public: + SampleProfileReaderItaniumRemapper(std::unique_ptr<MemoryBuffer> B, + std::unique_ptr<SymbolRemappingReader> SRR, + SampleProfileReader &R) + : Buffer(std::move(B)), Remappings(std::move(SRR)), Reader(R) { + assert(Remappings && "Remappings cannot be nullptr"); + } + + /// Create a remapper from the given remapping file. The remapper will + /// be used for profile read in by Reader. + static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> + create(const std::string Filename, SampleProfileReader &Reader, + LLVMContext &C); + + /// Create a remapper from the given Buffer. The remapper will + /// be used for profile read in by Reader. + static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> + create(std::unique_ptr<MemoryBuffer> &B, SampleProfileReader &Reader, + LLVMContext &C); + + /// Apply remappings to the profile read by Reader. + void applyRemapping(LLVMContext &Ctx); + + bool hasApplied() { return RemappingApplied; } + + /// Insert function name into remapper. + void insert(StringRef FunctionName) { Remappings->insert(FunctionName); } + + /// Query whether there is equivalent in the remapper which has been + /// inserted. + bool exist(StringRef FunctionName) { + return Remappings->lookup(FunctionName); + } + + /// Return the samples collected for function \p F if remapper knows + /// it is present in SampleMap. + FunctionSamples *getSamplesFor(StringRef FunctionName); + +private: + // The buffer holding the content read from remapping file. + std::unique_ptr<MemoryBuffer> Buffer; + std::unique_ptr<SymbolRemappingReader> Remappings; + DenseMap<SymbolRemappingReader::Key, FunctionSamples *> SampleMap; + // The Reader the remapper is servicing. + SampleProfileReader &Reader; + // Indicate whether remapping has been applied to the profile read + // by Reader -- by calling applyRemapping. + bool RemappingApplied = false; +}; + /// Sample-based profile reader. /// /// Each profile contains sample counts for all the functions @@ -273,13 +329,22 @@ public: /// Read and validate the file header. virtual std::error_code readHeader() = 0; - /// Read sample profiles from the associated file. - virtual std::error_code read() = 0; + /// The interface to read sample profiles from the associated file. + std::error_code read() { + if (std::error_code EC = readImpl()) + return EC; + if (Remapper) + Remapper->applyRemapping(Ctx); + return sampleprof_error::success; + } + + /// The implementaion to read sample profiles from the associated file. + virtual std::error_code readImpl() = 0; /// Print the profile for \p FName on stream \p OS. void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs()); - virtual void collectFuncsToUse(const Module &M) {} + virtual void collectFuncsFrom(const Module &M) {} /// Print all the profiles on stream \p OS. void dump(raw_ostream &OS = dbgs()); @@ -295,6 +360,10 @@ public: /// Return the samples collected for function \p F. virtual FunctionSamples *getSamplesFor(StringRef Fname) { + if (Remapper) { + if (auto FS = Remapper->getSamplesFor(Fname)) + return FS; + } std::string FGUID; Fname = getRepInFormat(Fname, getFormat(), FGUID); auto It = Profiles.find(Fname); @@ -313,18 +382,33 @@ public: } /// Create a sample profile reader appropriate to the file format. + /// Create a remapper underlying if RemapFilename is not empty. static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(const Twine &Filename, LLVMContext &C); + create(const std::string Filename, LLVMContext &C, + const std::string RemapFilename = ""); /// Create a sample profile reader from the supplied memory buffer. + /// Create a remapper underlying if RemapFilename is not empty. static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C); + create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, + const std::string RemapFilename = ""); /// Return the profile summary. - ProfileSummary &getSummary() { return *(Summary.get()); } + ProfileSummary &getSummary() const { return *(Summary.get()); } + + MemoryBuffer *getBuffer() const { return Buffer.get(); } /// \brief Return the profile format. - SampleProfileFormat getFormat() { return Format; } + SampleProfileFormat getFormat() const { return Format; } + + virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() { + return nullptr; + }; + + /// It includes all the names that have samples either in outline instance + /// or inline instance. + virtual std::vector<StringRef> *getNameTable() { return nullptr; } + virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; }; protected: /// Map every function to its associated profile. @@ -352,6 +436,8 @@ protected: /// Compute summary for this profile. void computeSummary(); + std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; @@ -365,7 +451,7 @@ public: std::error_code readHeader() override { return sampleprof_error::success; } /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -381,7 +467,11 @@ public: virtual std::error_code readHeader() override; /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; + + /// It includes all the names that have samples either in outline instance + /// or inline instance. + virtual std::vector<StringRef> *getNameTable() override { return &NameTable; } protected: /// Read a numeric value of type T from the profile. @@ -411,46 +501,134 @@ protected: bool at_eof() const { return Data >= End; } /// Read the next function profile instance. - std::error_code readFuncProfile(); + std::error_code readFuncProfile(const uint8_t *Start); /// Read the contents of the given profile instance. std::error_code readProfile(FunctionSamples &FProfile); + /// Read the contents of Magic number and Version number. + std::error_code readMagicIdent(); + + /// Read profile summary. + std::error_code readSummary(); + + /// Read the whole name table. + virtual std::error_code readNameTable(); + /// Points to the current location in the buffer. const uint8_t *Data = nullptr; /// Points to the end of the buffer. const uint8_t *End = nullptr; + /// Function name table. + std::vector<StringRef> NameTable; + + /// Read a string indirectly via the name table. + virtual ErrorOr<StringRef> readStringFromTable(); + private: std::error_code readSummaryEntry(std::vector<ProfileSummaryEntry> &Entries); virtual std::error_code verifySPMagic(uint64_t Magic) = 0; +}; - /// Read profile summary. - std::error_code readSummary(); +class SampleProfileReaderRawBinary : public SampleProfileReaderBinary { +private: + virtual std::error_code verifySPMagic(uint64_t Magic) override; - /// Read the whole name table. - virtual std::error_code readNameTable() = 0; +public: + SampleProfileReaderRawBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C, + SampleProfileFormat Format = SPF_Binary) + : SampleProfileReaderBinary(std::move(B), C, Format) {} - /// Read a string indirectly via the name table. - virtual ErrorOr<StringRef> readStringFromTable() = 0; + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); }; -class SampleProfileReaderRawBinary : public SampleProfileReaderBinary { +/// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase defines +/// the basic structure of the extensible binary format. +/// The format is organized in sections except the magic and version number +/// at the beginning. There is a section table before all the sections, and +/// each entry in the table describes the entry type, start, size and +/// attributes. The format in each section is defined by the section itself. +/// +/// It is easy to add a new section while maintaining the backward +/// compatibility of the profile. Nothing extra needs to be done. If we want +/// to extend an existing section, like add cache misses information in +/// addition to the sample count in the profile body, we can add a new section +/// with the extension and retire the existing section, and we could choose +/// to keep the parser of the old section if we want the reader to be able +/// to read both new and old format profile. +/// +/// SampleProfileReaderExtBinary/SampleProfileWriterExtBinary define the +/// commonly used sections of a profile in extensible binary format. It is +/// possible to define other types of profile inherited from +/// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase. +class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary { +private: + std::error_code decompressSection(const uint8_t *SecStart, + const uint64_t SecSize, + const uint8_t *&DecompressBuf, + uint64_t &DecompressBufSize); + + BumpPtrAllocator Allocator; + +protected: + std::vector<SecHdrTableEntry> SecHdrTable; + std::unique_ptr<ProfileSymbolList> ProfSymList; + std::error_code readSecHdrTableEntry(); + std::error_code readSecHdrTable(); + virtual std::error_code readHeader() override; + virtual std::error_code verifySPMagic(uint64_t Magic) override = 0; + virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, + SecType Type) = 0; + +public: + SampleProfileReaderExtBinaryBase(std::unique_ptr<MemoryBuffer> B, + LLVMContext &C, SampleProfileFormat Format) + : SampleProfileReaderBinary(std::move(B), C, Format) {} + + /// Read sample profiles in extensible format from the associated file. + std::error_code readImpl() override; + + /// Get the total size of all \p Type sections. + uint64_t getSectionSize(SecType Type); + /// Get the total size of header and all sections. + uint64_t getFileSize(); + virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) override; +}; + +class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { private: - /// Function name table. - std::vector<StringRef> NameTable; virtual std::error_code verifySPMagic(uint64_t Magic) override; - virtual std::error_code readNameTable() override; - /// Read a string indirectly via the name table. - virtual ErrorOr<StringRef> readStringFromTable() override; + virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, + SecType Type) override; + std::error_code readProfileSymbolList(); + std::error_code readFuncOffsetTable(); + std::error_code readFuncProfiles(); + + /// The table mapping from function name to the offset of its FunctionSample + /// towards file start. + DenseMap<StringRef, uint64_t> FuncOffsetTable; + /// The set containing the functions to use when compiling a module. + DenseSet<StringRef> FuncsToUse; + /// Use all functions from the input profile. + bool UseAllFuncs = true; public: - SampleProfileReaderRawBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C) - : SampleProfileReaderBinary(std::move(B), C, SPF_Binary) {} + SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C, + SampleProfileFormat Format = SPF_Ext_Binary) + : SampleProfileReaderExtBinaryBase(std::move(B), C, Format) {} /// \brief Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); + + virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override { + return std::move(ProfSymList); + }; + + /// Collect functions with definitions in Module \p M. + void collectFuncsFrom(const Module &M) override; }; class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { @@ -462,6 +640,8 @@ private: DenseMap<StringRef, uint64_t> FuncOffsetTable; /// The set containing the functions to use when compiling a module. DenseSet<StringRef> FuncsToUse; + /// Use all functions from the input profile. + bool UseAllFuncs = true; virtual std::error_code verifySPMagic(uint64_t Magic) override; virtual std::error_code readNameTable() override; /// Read a string indirectly via the name table. @@ -478,10 +658,10 @@ public: static bool hasFormat(const MemoryBuffer &Buffer); /// Read samples only for functions to use. - std::error_code read() override; + std::error_code readImpl() override; /// Collect functions to be used when compiling Module \p M. - void collectFuncsToUse(const Module &M) override; + void collectFuncsFrom(const Module &M) override; }; using InlineCallStack = SmallVector<FunctionSamples *, 10>; @@ -509,7 +689,7 @@ public: std::error_code readHeader() override; /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -537,44 +717,6 @@ protected: static const uint32_t GCOVTagAFDOFunction = 0xac000000; }; -/// A profile data reader proxy that remaps the profile data from another -/// sample profile data reader, by applying a provided set of equivalences -/// between components of the symbol names in the profile. -class SampleProfileReaderItaniumRemapper : public SampleProfileReader { -public: - SampleProfileReaderItaniumRemapper( - std::unique_ptr<MemoryBuffer> B, LLVMContext &C, - std::unique_ptr<SampleProfileReader> Underlying) - : SampleProfileReader(std::move(B), C, Underlying->getFormat()) { - Profiles = std::move(Underlying->getProfiles()); - Summary = takeSummary(*Underlying); - // Keep the underlying reader alive; the profile data may contain - // StringRefs referencing names in its name table. - UnderlyingReader = std::move(Underlying); - } - - /// Create a remapped sample profile from the given remapping file and - /// underlying samples. - static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(const Twine &Filename, LLVMContext &C, - std::unique_ptr<SampleProfileReader> Underlying); - - /// Read and validate the file header. - std::error_code readHeader() override { return sampleprof_error::success; } - - /// Read remapping file and apply it to the sample profile. - std::error_code read() override; - - /// Return the samples collected for function \p F. - FunctionSamples *getSamplesFor(StringRef FunctionName) override; - using SampleProfileReader::getSamplesFor; - -private: - SymbolRemappingReader Remappings; - DenseMap<SymbolRemappingReader::Key, FunctionSamples*> SampleMap; - std::unique_ptr<SampleProfileReader> UnderlyingReader; -}; - } // end namespace sampleprof } // end namespace llvm diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index 81e6e3ab0b4a..cc951594c9e2 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -36,7 +36,7 @@ public: /// Write sample profiles in \p S. /// /// \returns status code of the file update operation. - virtual std::error_code write(const FunctionSamples &S) = 0; + virtual std::error_code writeSample(const FunctionSamples &S) = 0; /// Write all the sample profiles in the given map of samples. /// @@ -56,6 +56,8 @@ public: static ErrorOr<std::unique_ptr<SampleProfileWriter>> create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); + virtual void setProfileSymbolList(ProfileSymbolList *PSL) {} + protected: SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) : OutputStream(std::move(OS)) {} @@ -64,6 +66,10 @@ protected: virtual std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) = 0; + // Write function profiles to the profile file. + virtual std::error_code + writeFuncProfiles(const StringMap<FunctionSamples> &ProfileMap); + /// Output stream where to emit the profile to. std::unique_ptr<raw_ostream> OutputStream; @@ -72,12 +78,15 @@ protected: /// Compute summary for this profile. void computeSummary(const StringMap<FunctionSamples> &ProfileMap); + + /// Profile format. + SampleProfileFormat Format; }; /// Sample-based profile writer (text format). class SampleProfileWriterText : public SampleProfileWriter { public: - std::error_code write(const FunctionSamples &S) override; + std::error_code writeSample(const FunctionSamples &S) override; protected: SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) @@ -102,13 +111,14 @@ private: /// Sample-based profile writer (binary format). class SampleProfileWriterBinary : public SampleProfileWriter { public: - virtual std::error_code write(const FunctionSamples &S) override; SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) : SampleProfileWriter(OS) {} + virtual std::error_code writeSample(const FunctionSamples &S) override; + protected: - virtual std::error_code writeNameTable() = 0; - virtual std::error_code writeMagicIdent() = 0; + virtual std::error_code writeMagicIdent(SampleProfileFormat Format); + virtual std::error_code writeNameTable(); virtual std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; std::error_code writeSummary(); @@ -118,10 +128,10 @@ protected: MapVector<StringRef, uint32_t> NameTable; -private: void addName(StringRef FName); void addNames(const FunctionSamples &S); +private: friend ErrorOr<std::unique_ptr<SampleProfileWriter>> SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); @@ -129,10 +139,99 @@ private: class SampleProfileWriterRawBinary : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; +}; + +class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { + using SampleProfileWriterBinary::SampleProfileWriterBinary; +public: + virtual std::error_code + write(const StringMap<FunctionSamples> &ProfileMap) override; + + void setToCompressAllSections(); + void setToCompressSection(SecType Type); protected: - virtual std::error_code writeNameTable() override; - virtual std::error_code writeMagicIdent() override; + uint64_t markSectionStart(SecType Type); + std::error_code addNewSection(SecType Sec, uint64_t SectionStart); + virtual void initSectionHdrLayout() = 0; + virtual std::error_code + writeSections(const StringMap<FunctionSamples> &ProfileMap) = 0; + + // Specifiy the order of sections in section header table. Note + // the order of sections in the profile may be different that the + // order in SectionHdrLayout. sample Reader will follow the order + // in SectionHdrLayout to read each section. + SmallVector<SecHdrTableEntry, 8> SectionHdrLayout; + +private: + void allocSecHdrTable(); + std::error_code writeSecHdrTable(); + virtual std::error_code + writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; + void addSectionFlags(SecType Type, SecFlags Flags); + SecHdrTableEntry &getEntryInLayout(SecType Type); + std::error_code compressAndOutput(); + + // We will swap the raw_ostream held by LocalBufStream and that + // held by OutputStream if we try to add a section which needs + // compression. After the swap, all the data written to output + // will be temporarily buffered into the underlying raw_string_ostream + // originally held by LocalBufStream. After the data writing for the + // section is completed, compress the data in the local buffer, + // swap the raw_ostream back and write the compressed data to the + // real output. + std::unique_ptr<raw_ostream> LocalBufStream; + // The location where the output stream starts. + uint64_t FileStart; + // The location in the output stream where the SecHdrTable should be + // written to. + uint64_t SecHdrTableOffset; + // Initial Section Flags setting. + std::vector<SecHdrTableEntry> SecHdrTable; +}; + +class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { +public: + SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS) + : SampleProfileWriterExtBinaryBase(OS) { + initSectionHdrLayout(); + } + + virtual std::error_code writeSample(const FunctionSamples &S) override; + virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { + ProfSymList = PSL; + }; + +private: + virtual void initSectionHdrLayout() override { + // Note that SecFuncOffsetTable section is written after SecLBRProfile + // in the profile, but is put before SecLBRProfile in SectionHdrLayout. + // + // This is because sample reader follows the order of SectionHdrLayout to + // read each section, to read function profiles on demand sample reader + // need to get the offset of each function profile first. + // + // SecFuncOffsetTable section is written after SecLBRProfile in the + // profile because FuncOffsetTable needs to be populated while section + // SecLBRProfile is written. + SectionHdrLayout = {{SecProfSummary, 0, 0, 0}, + {SecNameTable, 0, 0, 0}, + {SecFuncOffsetTable, 0, 0, 0}, + {SecLBRProfile, 0, 0, 0}, + {SecProfileSymbolList, 0, 0, 0}}; + }; + virtual std::error_code + writeSections(const StringMap<FunctionSamples> &ProfileMap) override; + ProfileSymbolList *ProfSymList = nullptr; + + // Save the start of SecLBRProfile so we can compute the offset to the + // start of SecLBRProfile for each Function's Profile and will keep it + // in FuncOffsetTable. + uint64_t SecLBRProfileStart; + // FuncOffsetTable maps function name to its profile offset in SecLBRProfile + // section. It is used to load function profile on demand. + MapVector<StringRef, uint64_t> FuncOffsetTable; + std::error_code writeFuncOffsetTable(); }; // CompactBinary is a compact format of binary profile which both reduces @@ -169,7 +268,7 @@ class SampleProfileWriterCompactBinary : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; public: - virtual std::error_code write(const FunctionSamples &S) override; + virtual std::error_code writeSample(const FunctionSamples &S) override; virtual std::error_code write(const StringMap<FunctionSamples> &ProfileMap) override; @@ -181,7 +280,6 @@ protected: /// towards profile start. uint64_t TableOffset; virtual std::error_code writeNameTable() override; - virtual std::error_code writeMagicIdent() override; virtual std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; std::error_code writeFuncOffsetTable(); |