diff options
Diffstat (limited to 'include/llvm/ProfileData')
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMapping.h (renamed from include/llvm/ProfileData/CoverageMapping.h) | 126 | ||||
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMappingReader.h (renamed from include/llvm/ProfileData/CoverageMappingReader.h) | 41 | ||||
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMappingWriter.h (renamed from include/llvm/ProfileData/CoverageMappingWriter.h) | 2 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProf.h | 545 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfData.inc | 338 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfReader.h | 135 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfWriter.h | 30 | ||||
| -rw-r--r-- | include/llvm/ProfileData/ProfileCommon.h | 102 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProf.h | 44 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProfReader.h | 42 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProfWriter.h | 22 |
11 files changed, 938 insertions, 489 deletions
diff --git a/include/llvm/ProfileData/CoverageMapping.h b/include/llvm/ProfileData/Coverage/CoverageMapping.h index 92a991eb39bab..6afde56122f00 100644 --- a/include/llvm/ProfileData/CoverageMapping.h +++ b/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -23,13 +23,13 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/raw_ostream.h" #include <system_error> #include <tuple> namespace llvm { namespace coverage { + enum class coveragemap_error { success = 0, eof, @@ -38,14 +38,37 @@ enum class coveragemap_error { truncated, malformed }; -} // end of coverage namespace. + +const std::error_category &coveragemap_category(); + +inline std::error_code make_error_code(coveragemap_error E) { + return std::error_code(static_cast<int>(E), coveragemap_category()); } -namespace std { -template <> -struct is_error_code_enum<llvm::coverage::coveragemap_error> : std::true_type { +class CoverageMapError : public ErrorInfo<CoverageMapError> { +public: + CoverageMapError(coveragemap_error Err) : Err(Err) { + assert(Err != coveragemap_error::success && "Not an error"); + } + + std::string message() const override; + + void log(raw_ostream &OS) const override { OS << message(); } + + std::error_code convertToErrorCode() const override { + return make_error_code(Err); + } + + coveragemap_error get() const { return Err; } + + static char ID; + +private: + coveragemap_error Err; }; -} + +} // end of coverage namespace. +} // end of llvm namespace namespace llvm { class IndexedInstrProfReader; @@ -265,7 +288,7 @@ public: /// \brief Return the number of times that a region of code associated with /// this counter was executed. - ErrorOr<int64_t> evaluate(const Counter &C) const; + Expected<int64_t> evaluate(const Counter &C) const; }; /// \brief Code coverage information for a single function. @@ -370,13 +393,6 @@ struct CoverageSegment { return std::tie(L.Line, L.Col, L.Count, L.HasCount, L.IsRegionEntry) == std::tie(R.Line, R.Col, R.Count, R.HasCount, R.IsRegionEntry); } - - void setCount(uint64_t NewCount) { - Count = NewCount; - HasCount = true; - } - - void addCount(uint64_t NewCount) { setCount(Count + NewCount); } }; /// \brief Coverage information to be processed or displayed. @@ -400,14 +416,14 @@ public: Expansions(std::move(RHS.Expansions)) {} /// \brief Get the name of the file this data covers. - StringRef getFilename() { return Filename; } + StringRef getFilename() const { return Filename; } std::vector<CoverageSegment>::iterator begin() { return Segments.begin(); } std::vector<CoverageSegment>::iterator end() { return Segments.end(); } bool empty() { return Segments.empty(); } /// \brief Expansions that can be further processed. - std::vector<ExpansionRecord> getExpansions() { return Expansions; } + ArrayRef<ExpansionRecord> getExpansions() { return Expansions; } }; /// \brief The mapping of profile information to coverage data. @@ -422,12 +438,12 @@ class CoverageMapping { public: /// \brief Load the coverage mapping using the given readers. - static ErrorOr<std::unique_ptr<CoverageMapping>> + static Expected<std::unique_ptr<CoverageMapping>> load(CoverageMappingReader &CoverageReader, IndexedInstrProfReader &ProfileReader); /// \brief Load the coverage mapping from the given files. - static ErrorOr<std::unique_ptr<CoverageMapping>> + static Expected<std::unique_ptr<CoverageMapping>> load(StringRef ObjectFilename, StringRef ProfileFilename, StringRef Arch = StringRef()); @@ -445,7 +461,7 @@ public: /// The given filename must be the name as recorded in the coverage /// information. That is, only names returned from getUniqueSourceFiles will /// yield a result. - CoverageData getCoverageForFile(StringRef Filename); + CoverageData getCoverageForFile(StringRef Filename) const; /// \brief Gets all of the functions covered by this profile. iterator_range<FunctionRecordIterator> getCoveredFunctions() const { @@ -464,21 +480,16 @@ public: /// /// Functions that are instantiated more than once, such as C++ template /// specializations, have distinct coverage records for each instantiation. - std::vector<const FunctionRecord *> getInstantiations(StringRef Filename); + std::vector<const FunctionRecord *> + getInstantiations(StringRef Filename) const; /// \brief Get the coverage for a particular function. - CoverageData getCoverageForFunction(const FunctionRecord &Function); + CoverageData getCoverageForFunction(const FunctionRecord &Function) const; /// \brief Get the coverage for an expansion within a coverage set. - CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion); + CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion) const; }; -const std::error_category &coveragemap_category(); - -inline std::error_code make_error_code(coveragemap_error E) { - return std::error_code(static_cast<int>(E), coveragemap_category()); -} - // Profile coverage map has the following layout: // [CoverageMapFileHeader] // [ArrayStart] @@ -488,9 +499,11 @@ inline std::error_code make_error_code(coveragemap_error E) { // [ArrayEnd] // [Encoded Region Mapping Data] LLVM_PACKED_START -template <class IntPtrT> struct CovMapFunctionRecord { +template <class IntPtrT> struct CovMapFunctionRecordV1 { +#define COVMAP_V1 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name; #include "llvm/ProfileData/InstrProfData.inc" +#undef COVMAP_V1 // Return the structural hash associated with the function. template <support::endianness Endian> uint64_t getFuncHash() const { @@ -506,16 +519,41 @@ template <class IntPtrT> struct CovMapFunctionRecord { } // Return the PGO name of the function */ template <support::endianness Endian> - std::error_code getFuncName(InstrProfSymtab &ProfileNames, - StringRef &FuncName) const { + Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const { IntPtrT NameRef = getFuncNameRef<Endian>(); uint32_t NameS = support::endian::byte_swap<uint32_t, Endian>(NameSize); FuncName = ProfileNames.getFuncName(NameRef, NameS); if (NameS && FuncName.empty()) - return coveragemap_error::malformed; - return std::error_code(); + return make_error<CoverageMapError>(coveragemap_error::malformed); + return Error::success(); + } +}; + +struct CovMapFunctionRecord { +#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" + + // Return the structural hash associated with the function. + template <support::endianness Endian> uint64_t getFuncHash() const { + return support::endian::byte_swap<uint64_t, Endian>(FuncHash); + } + // Return the coverage map data size for the funciton. + template <support::endianness Endian> uint32_t getDataSize() const { + return support::endian::byte_swap<uint32_t, Endian>(DataSize); + } + // Return function lookup key. The value is consider opaque. + template <support::endianness Endian> uint64_t getFuncNameRef() const { + return support::endian::byte_swap<uint64_t, Endian>(NameRef); + } + // Return the PGO name of the function */ + template <support::endianness Endian> + Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const { + uint64_t NameRef = getFuncNameRef<Endian>(); + FuncName = ProfileNames.getFuncName(NameRef); + return Error::success(); } }; + // Per module coverage mapping data header, i.e. CoverageMapFileHeader // documented above. struct CovMapHeader { @@ -537,10 +575,24 @@ struct CovMapHeader { LLVM_PACKED_END -enum CoverageMappingVersion { - CoverageMappingVersion1 = 0, - // The current versin is Version1 - CoverageMappingCurrentVersion = INSTR_PROF_COVMAP_VERSION +enum CovMapVersion { + Version1 = 0, + // Function's name reference from CovMapFuncRecord is changed from raw + // name string pointer to MD5 to support name section compression. Name + // section is also compressed. + Version2 = 1, + // The current version is Version2 + CurrentVersion = INSTR_PROF_COVMAP_VERSION +}; + +template <int CovMapVersion, class IntPtrT> struct CovMapTraits { + typedef CovMapFunctionRecord CovMapFuncRecordType; + typedef uint64_t NameRefType; +}; + +template <class IntPtrT> struct CovMapTraits<CovMapVersion::Version1, IntPtrT> { + typedef CovMapFunctionRecordV1<IntPtrT> CovMapFuncRecordType; + typedef IntPtrT NameRefType; }; } // end namespace coverage diff --git a/include/llvm/ProfileData/CoverageMappingReader.h b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 38fb4680476be..db907f128d931 100644 --- a/include/llvm/ProfileData/CoverageMappingReader.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -19,7 +19,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/ProfileData/CoverageMapping.h" +#include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" @@ -69,7 +69,7 @@ public: class CoverageMappingReader { public: - virtual std::error_code readNextRecord(CoverageMappingRecord &Record) = 0; + virtual Error readNextRecord(CoverageMappingRecord &Record) = 0; CoverageMappingIterator begin() { return CoverageMappingIterator(this); } CoverageMappingIterator end() { return CoverageMappingIterator(); } virtual ~CoverageMappingReader() {} @@ -82,10 +82,10 @@ protected: RawCoverageReader(StringRef Data) : Data(Data) {} - std::error_code readULEB128(uint64_t &Result); - std::error_code readIntMax(uint64_t &Result, uint64_t MaxPlus1); - std::error_code readSize(uint64_t &Result); - std::error_code readString(StringRef &Result); + Error readULEB128(uint64_t &Result); + Error readIntMax(uint64_t &Result, uint64_t MaxPlus1); + Error readSize(uint64_t &Result); + Error readString(StringRef &Result); }; /// \brief Reader for the raw coverage filenames. @@ -100,7 +100,17 @@ public: RawCoverageFilenamesReader(StringRef Data, std::vector<StringRef> &Filenames) : RawCoverageReader(Data), Filenames(Filenames) {} - std::error_code read(); + Error read(); +}; + +/// \brief Checks if the given coverage mapping data is exported for +/// an unused function. +class RawCoverageMappingDummyChecker : public RawCoverageReader { +public: + RawCoverageMappingDummyChecker(StringRef MappingData) + : RawCoverageReader(MappingData) {} + + Expected<bool> isDummy(); }; /// \brief Reader for the raw coverage mapping data. @@ -125,12 +135,12 @@ public: Filenames(Filenames), Expressions(Expressions), MappingRegions(MappingRegions) {} - std::error_code read(); + Error read(); private: - std::error_code decodeCounter(unsigned Value, Counter &C); - std::error_code readCounter(Counter &C); - std::error_code + Error decodeCounter(unsigned Value, Counter &C); + Error readCounter(Counter &C); + Error readMappingRegionsSubArray(std::vector<CounterMappingRegion> &MappingRegions, unsigned InferredFileID, size_t NumFileIDs); }; @@ -140,14 +150,14 @@ private: class BinaryCoverageReader : public CoverageMappingReader { public: struct ProfileMappingRecord { - CoverageMappingVersion Version; + CovMapVersion Version; StringRef FunctionName; uint64_t FunctionHash; StringRef CoverageMapping; size_t FilenamesBegin; size_t FilenamesSize; - ProfileMappingRecord(CoverageMappingVersion Version, StringRef FunctionName, + ProfileMappingRecord(CovMapVersion Version, StringRef FunctionName, uint64_t FunctionHash, StringRef CoverageMapping, size_t FilenamesBegin, size_t FilenamesSize) : Version(Version), FunctionName(FunctionName), @@ -158,6 +168,7 @@ public: private: std::vector<StringRef> Filenames; std::vector<ProfileMappingRecord> MappingRecords; + InstrProfSymtab ProfileNames; size_t CurrentRecord; std::vector<StringRef> FunctionsFilenames; std::vector<CounterExpression> Expressions; @@ -169,11 +180,11 @@ private: BinaryCoverageReader() : CurrentRecord(0) {} public: - static ErrorOr<std::unique_ptr<BinaryCoverageReader>> + static Expected<std::unique_ptr<BinaryCoverageReader>> create(std::unique_ptr<MemoryBuffer> &ObjectBuffer, StringRef Arch); - std::error_code readNextRecord(CoverageMappingRecord &Record) override; + Error readNextRecord(CoverageMappingRecord &Record) override; }; } // end namespace coverage diff --git a/include/llvm/ProfileData/CoverageMappingWriter.h b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h index 2e3b0378d032f..10269cc50f353 100644 --- a/include/llvm/ProfileData/CoverageMappingWriter.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h @@ -17,7 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ProfileData/CoverageMapping.h" +#include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/Support/raw_ostream.h" namespace llvm { diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index c84d8d24f2061..75646b7616592 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -1,4 +1,4 @@ -//=-- InstrProf.h - Instrumented profiling format support ---------*- C++ -*-=// +//===-- InstrProf.h - Instrumented profiling format support -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -13,18 +13,20 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_PROFILEDATA_INSTRPROF_H_ -#define LLVM_PROFILEDATA_INSTRPROF_H_ +#ifndef LLVM_PROFILEDATA_INSTRPROF_H +#define LLVM_PROFILEDATA_INSTRPROF_H #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/IR/Metadata.h" #include "llvm/ProfileData/InstrProfData.inc" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/Support/Endian.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/MD5.h" +#include "llvm/Support/MathExtras.h" #include <cstdint> #include <list> #include <system_error> @@ -56,6 +58,20 @@ inline StringRef getInstrProfDataSectionName(bool AddSegment) { : INSTR_PROF_DATA_SECT_NAME_STR; } +/// Return the name of data section containing pointers to value profile +/// counters/nodes. +inline StringRef getInstrProfValuesSectionName(bool AddSegment) { + return AddSegment ? "__DATA," INSTR_PROF_VALS_SECT_NAME_STR + : INSTR_PROF_VALS_SECT_NAME_STR; +} + +/// Return the name of data section containing nodes holdling value +/// profiling data. +inline StringRef getInstrProfVNodesSectionName(bool AddSegment) { + return AddSegment ? "__DATA," INSTR_PROF_VNODES_SECT_NAME_STR + : INSTR_PROF_VNODES_SECT_NAME_STR; +} + /// Return the name profile runtime entry point to do value profiling /// for a given site. inline StringRef getInstrProfValueProfFuncName() { @@ -78,10 +94,22 @@ inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } /// Return the name prefix of profile counter variables. inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } +/// Return the name prefix of value profile variables. +inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } + +/// Return the name of value profile node array variables: +inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } + /// Return the name prefix of the COMDAT group for instrumentation variables /// associated with a COMDAT function. inline StringRef getInstrProfComdatPrefix() { return "__profv_"; } +/// Return the name of the variable holding the strings (possibly compressed) +/// of all function's PGO names. +inline StringRef getInstrProfNamesVarName() { + return "__llvm_prf_nm"; +} + /// Return the name of a covarage mapping variable (internal linkage) /// for each instrumented source module. Such variables are allocated /// in the __llvm_covmap section. @@ -90,10 +118,12 @@ inline StringRef getCoverageMappingVarName() { } /// Return the name of the internal variable recording the array -/// of PGO name vars referenced by the coverage mapping, The owning +/// of PGO name vars referenced by the coverage mapping. The owning /// functions of those names are not emitted by FE (e.g, unused inline /// functions.) -inline StringRef getCoverageNamesVarName() { return "__llvm_coverage_names"; } +inline StringRef getCoverageUnusedNamesVarName() { + return "__llvm_coverage_names"; +} /// Return the name of function that registers all the per-function control /// data at program startup time by calling __llvm_register_function. This @@ -110,6 +140,11 @@ inline StringRef getInstrProfRegFuncName() { return "__llvm_profile_register_function"; } +/// Return the name of the runtime interface that registers the PGO name strings. +inline StringRef getInstrProfNamesRegFuncName() { + return "__llvm_profile_register_names_function"; +} + /// Return the name of the runtime initialization method that is generated by /// the compiler. The function calls __llvm_profile_register_functions and /// __llvm_profile_override_default_filename functions if needed. This function @@ -135,9 +170,13 @@ inline StringRef getInstrProfFileOverriderFuncName() { return "__llvm_profile_override_default_filename"; } +/// Return the marker used to separate PGO names during serialization. +inline StringRef getInstrProfNameSeparator() { return "\01"; } + /// Return the modified name for function \c F suitable to be -/// used the key for profile lookup. -std::string getPGOFuncName(const Function &F, +/// used the key for profile lookup. Variable \c InLTO indicates if this +/// is called in LTO optimization passes. +std::string getPGOFuncName(const Function &F, bool InLTO = false, uint64_t Version = INSTR_PROF_INDEX_VERSION); /// Return the modified name for a function suitable to be @@ -149,10 +188,16 @@ std::string getPGOFuncName(StringRef RawFuncName, StringRef FileName, uint64_t Version = INSTR_PROF_INDEX_VERSION); +/// Return the name of the global variable used to store a function +/// name in PGO instrumentation. \c FuncName is the name of the function +/// returned by the \c getPGOFuncName call. +std::string getPGOFuncNameVarName(StringRef FuncName, + GlobalValue::LinkageTypes Linkage); + /// Create and return the global variable for function name used in PGO /// instrumentation. \c FuncName is the name of the function returned /// by \c getPGOFuncName call. -GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName); +GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); /// Create and return the global variable for function name used in PGO /// instrumentation. /// \c FuncName is the name of the function @@ -160,13 +205,14 @@ GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName); /// and \c Linkage is the linkage of the instrumented function. GlobalVariable *createPGOFuncNameVar(Module &M, GlobalValue::LinkageTypes Linkage, - StringRef FuncName); + StringRef PGOFuncName); /// Return the initializer in string of the PGO name var \c NameVar. StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar); /// Given a PGO function name, remove the filename prefix and return /// the original (static) function name. -StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName); +StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, + StringRef FileName = "<unknown>"); /// Given a vector of strings (function PGO names) \c NameStrs, the /// method generates a combined string \c Result thatis ready to be @@ -174,22 +220,59 @@ StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName); /// The first field is the legnth of the uncompressed strings, and the /// the second field is the length of the zlib-compressed string. /// Both fields are encoded in ULEB128. If \c doCompress is false, the -/// third field is the uncompressed strings; otherwise it is the -/// compressed string. When the string compression is off, the +/// third field is the uncompressed strings; otherwise it is the +/// compressed string. When the string compression is off, the /// second field will have value zero. -int collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs, - bool doCompression, std::string &Result); +Error collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs, + bool doCompression, std::string &Result); /// Produce \c Result string with the same format described above. The input /// is vector of PGO function name variables that are referenced. -int collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars, - std::string &Result); +Error collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars, + std::string &Result, bool doCompression = true); class InstrProfSymtab; /// \c NameStrings is a string composed of one of more sub-strings encoded in -/// the -/// format described above. The substrings are seperated by 0 or more zero -/// bytes. -/// This method decodes the string and populates the \c Symtab. -int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab); +/// the format described above. The substrings are seperated by 0 or more zero +/// bytes. This method decodes the string and populates the \c Symtab. +Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab); + +enum InstrProfValueKind : uint32_t { +#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#include "llvm/ProfileData/InstrProfData.inc" +}; + +struct InstrProfRecord; + +/// Get the value profile data for value site \p SiteIdx from \p InstrProfR +/// and annotate the instruction \p Inst with the value profile meta data. +/// Annotate up to \p MaxMDCount (default 3) number of records per value site. +void annotateValueSite(Module &M, Instruction &Inst, + const InstrProfRecord &InstrProfR, + InstrProfValueKind ValueKind, uint32_t SiteIndx, + uint32_t MaxMDCount = 3); +/// Same as the above interface but using an ArrayRef, as well as \p Sum. +void annotateValueSite(Module &M, Instruction &Inst, + ArrayRef<InstrProfValueData> VDs, + uint64_t Sum, InstrProfValueKind ValueKind, + uint32_t MaxMDCount); + +/// Extract the value profile data from \p Inst which is annotated with +/// value profile meta data. Return false if there is no value data annotated, +/// otherwise return true. +bool getValueProfDataFromInst(const Instruction &Inst, + InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, uint64_t &TotalC); + +inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } + +/// Return the PGOFuncName meta data associated with a function. +MDNode *getPGOFuncNameMetadata(const Function &F); + +/// Create the PGOFuncName meta data if PGOFuncName is different from +/// function's raw name. This should only apply to internal linkage functions +/// declared by users only. +void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); const std::error_category &instrprof_category(); @@ -208,26 +291,104 @@ enum class instrprof_error { hash_mismatch, count_mismatch, counter_overflow, - value_site_count_mismatch + value_site_count_mismatch, + compress_failed, + uncompress_failed }; inline std::error_code make_error_code(instrprof_error E) { return std::error_code(static_cast<int>(E), instrprof_category()); } -inline instrprof_error MergeResult(instrprof_error &Accumulator, - instrprof_error Result) { - // Prefer first error encountered as later errors may be secondary effects of - // the initial problem. - if (Accumulator == instrprof_error::success && - Result != instrprof_error::success) - Accumulator = Result; - return Accumulator; -} +class InstrProfError : public ErrorInfo<InstrProfError> { +public: + InstrProfError(instrprof_error Err) : Err(Err) { + assert(Err != instrprof_error::success && "Not an error"); + } -enum InstrProfValueKind : uint32_t { -#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, -#include "llvm/ProfileData/InstrProfData.inc" + std::string message() const override; + + void log(raw_ostream &OS) const override { OS << message(); } + + std::error_code convertToErrorCode() const override { + return make_error_code(Err); + } + + instrprof_error get() const { return Err; } + + /// Consume an Error and return the raw enum value contained within it. The + /// Error must either be a success value, or contain a single InstrProfError. + static instrprof_error take(Error E) { + auto Err = instrprof_error::success; + handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) { + assert(Err == instrprof_error::success && "Multiple errors encountered"); + Err = IPE.get(); + }); + return Err; + } + + static char ID; + +private: + instrprof_error Err; +}; + +class SoftInstrProfErrors { + /// Count the number of soft instrprof_errors encountered and keep track of + /// the first such error for reporting purposes. + + /// The first soft error encountered. + instrprof_error FirstError; + + /// The number of hash mismatches. + unsigned NumHashMismatches; + + /// The number of count mismatches. + unsigned NumCountMismatches; + + /// The number of counter overflows. + unsigned NumCounterOverflows; + + /// The number of value site count mismatches. + unsigned NumValueSiteCountMismatches; + +public: + SoftInstrProfErrors() + : FirstError(instrprof_error::success), NumHashMismatches(0), + NumCountMismatches(0), NumCounterOverflows(0), + NumValueSiteCountMismatches(0) {} + + ~SoftInstrProfErrors() { + assert(FirstError == instrprof_error::success && + "Unchecked soft error encountered"); + } + + /// Track a soft error (\p IE) and increment its associated counter. + void addError(instrprof_error IE); + + /// Get the number of hash mismatches. + unsigned getNumHashMismatches() const { return NumHashMismatches; } + + /// Get the number of count mismatches. + unsigned getNumCountMismatches() const { return NumCountMismatches; } + + /// Get the number of counter overflows. + unsigned getNumCounterOverflows() const { return NumCounterOverflows; } + + /// Get the number of value site count mismatches. + unsigned getNumValueSiteCountMismatches() const { + return NumValueSiteCountMismatches; + } + + /// Return the first encountered error and reset FirstError to a success + /// value. + Error takeError() { + if (FirstError == instrprof_error::success) + return Error::success(); + auto E = make_error<InstrProfError>(FirstError); + FirstError = instrprof_error::success; + return E; + } }; namespace object { @@ -250,27 +411,41 @@ public: private: StringRef Data; uint64_t Address; - // A map from MD5 hash keys to function name strings. - std::vector<std::pair<uint64_t, std::string>> HashNameMap; + // Unique name strings. + StringSet<> NameTab; + // A map from MD5 keys to function name strings. + std::vector<std::pair<uint64_t, StringRef>> MD5NameMap; + // A map from MD5 keys to function define. We only populate this map + // when build the Symtab from a Module. + std::vector<std::pair<uint64_t, Function *>> MD5FuncMap; // A map from function runtime address to function name MD5 hash. // This map is only populated and used by raw instr profile reader. AddrHashMap AddrToMD5Map; public: - InstrProfSymtab() : Data(), Address(0), HashNameMap(), AddrToMD5Map() {} + InstrProfSymtab() + : Data(), Address(0), NameTab(), MD5NameMap(), MD5FuncMap(), + AddrToMD5Map() {} /// Create InstrProfSymtab from an object file section which - /// contains function PGO names that are uncompressed. - /// This interface is used by CoverageMappingReader. - std::error_code create(object::SectionRef &Section); + /// contains function PGO names. When section may contain raw + /// string data or string data in compressed form. This method + /// only initialize the symtab with reference to the data and + /// the section base address. The decompression will be delayed + /// until before it is used. See also \c create(StringRef) method. + Error create(object::SectionRef &Section); /// This interface is used by reader of CoverageMapping test /// format. - inline std::error_code create(StringRef D, uint64_t BaseAddr); + inline Error create(StringRef D, uint64_t BaseAddr); /// \c NameStrings is a string composed of one of more sub-strings - /// encoded in the format described above. The substrings are - /// seperated by 0 or more zero bytes. This method decodes the - /// string and populates the \c Symtab. - inline std::error_code create(StringRef NameStrings); + /// encoded in the format described in \c collectPGOFuncNameStrings. + /// This method is a wrapper to \c readPGOFuncNameStrings method. + inline Error create(StringRef NameStrings); + /// A wrapper interface to populate the PGO symtab with functions + /// decls from module \c M. This interface is used by transformation + /// passes such as indirect function call promotion. Variable \c InLTO + /// indicates if this is called from LTO optimization passes. + void create(Module &M, bool InLTO = false); /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. template <typename NameIterRange> void create(const NameIterRange &IterRange); @@ -282,8 +457,10 @@ public: /// Update the symtab by adding \p FuncName to the table. This interface /// is used by the raw and text profile readers. void addFuncName(StringRef FuncName) { - HashNameMap.push_back(std::make_pair( - IndexedInstrProf::ComputeHash(FuncName), FuncName.str())); + auto Ins = NameTab.insert(FuncName); + if (Ins.second) + MD5NameMap.push_back(std::make_pair( + IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); } /// Map a function address to its name's MD5 hash. This interface /// is only used by the raw profiler reader. @@ -298,32 +475,37 @@ public: /// Return function's PGO name from the name's md5 hash value. /// If not found, return an empty string. inline StringRef getFuncName(uint64_t FuncMD5Hash); + /// Return function from the name's md5 hash. Return nullptr if not found. + inline Function *getFunction(uint64_t FuncMD5Hash); + /// Return the function's original assembly name by stripping off + /// the prefix attached (to symbols with priviate linkage). For + /// global functions, it returns the same string as getFuncName. + inline StringRef getOrigFuncName(uint64_t FuncMD5Hash); + /// Return the name section data. + inline StringRef getNameData() const { return Data; } }; -std::error_code InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { +Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { Data = D; Address = BaseAddr; - return std::error_code(); + return Error::success(); } -std::error_code InstrProfSymtab::create(StringRef NameStrings) { - if (readPGOFuncNameStrings(NameStrings, *this)) - return make_error_code(instrprof_error::malformed); - return std::error_code(); +Error InstrProfSymtab::create(StringRef NameStrings) { + return readPGOFuncNameStrings(NameStrings, *this); } template <typename NameIterRange> void InstrProfSymtab::create(const NameIterRange &IterRange) { for (auto Name : IterRange) - HashNameMap.push_back( - std::make_pair(IndexedInstrProf::ComputeHash(Name), Name.str())); + addFuncName(Name); + finalizeSymtab(); } void InstrProfSymtab::finalizeSymtab() { - std::sort(HashNameMap.begin(), HashNameMap.end(), less_first()); - HashNameMap.erase(std::unique(HashNameMap.begin(), HashNameMap.end()), - HashNameMap.end()); + std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first()); + std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first()); std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first()); AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), AddrToMD5Map.end()); @@ -331,14 +513,34 @@ void InstrProfSymtab::finalizeSymtab() { StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) { auto Result = - std::lower_bound(HashNameMap.begin(), HashNameMap.end(), FuncMD5Hash, + std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash, [](const std::pair<uint64_t, std::string> &LHS, uint64_t RHS) { return LHS.first < RHS; }); - if (Result != HashNameMap.end()) + if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash) return Result->second; return StringRef(); } +Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { + auto Result = + std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash, + [](const std::pair<uint64_t, Function*> &LHS, + uint64_t RHS) { return LHS.first < RHS; }); + if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) + return Result->second; + return nullptr; +} + +// See also getPGOFuncName implementation. These two need to be +// matched. +StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { + StringRef PGOName = getFuncName(FuncMD5Hash); + size_t S = PGOName.find_first_of(':'); + if (S == StringRef::npos) + return PGOName; + return PGOName.drop_front(S + 1); +} + struct InstrProfValueSiteRecord { /// Value profiling data pairs at a given value site. std::list<InstrProfValueData> ValueData; @@ -360,19 +562,21 @@ struct InstrProfValueSiteRecord { /// Merge data from another InstrProfValueSiteRecord /// Optionally scale merged counts by \p Weight. - instrprof_error merge(InstrProfValueSiteRecord &Input, uint64_t Weight = 1); + void merge(SoftInstrProfErrors &SIPE, InstrProfValueSiteRecord &Input, + uint64_t Weight = 1); /// Scale up value profile data counts. - instrprof_error scale(uint64_t Weight); + void scale(SoftInstrProfErrors &SIPE, uint64_t Weight); }; /// Profiling information for a single function. struct InstrProfRecord { - InstrProfRecord() {} + InstrProfRecord() : SIPE() {} InstrProfRecord(StringRef Name, uint64_t Hash, std::vector<uint64_t> Counts) - : Name(Name), Hash(Hash), Counts(std::move(Counts)) {} + : Name(Name), Hash(Hash), Counts(std::move(Counts)), SIPE() {} StringRef Name; uint64_t Hash; std::vector<uint64_t> Counts; + SoftInstrProfErrors SIPE; typedef std::vector<std::pair<uint64_t, uint64_t>> ValueMapType; @@ -387,13 +591,17 @@ struct InstrProfRecord { /// site: Site. inline uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const; - /// Return the array of profiled values at \p Site. + /// Return the array of profiled values at \p Site. If \p TotalC + /// is not null, the total count of all target values at this site + /// will be stored in \c *TotalC. inline std::unique_ptr<InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, - uint64_t (*ValueMapper)(uint32_t, uint64_t) = 0) const; - inline void - getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, uint32_t Site, - uint64_t (*ValueMapper)(uint32_t, uint64_t) = 0) const; + uint64_t *TotalC = 0) const; + /// Get the target value/counts of kind \p ValueKind collected at site + /// \p Site and store the result in array \p Dest. Return the total + /// counts of all target values at this site. + inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, + uint32_t Site) const; /// Reserve space for NumValueSites sites. inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); /// Add ValueData for ValueKind at value Site. @@ -403,11 +611,11 @@ struct InstrProfRecord { /// Merge the counts in \p Other into this one. /// Optionally scale merged counts by \p Weight. - instrprof_error merge(InstrProfRecord &Other, uint64_t Weight = 1); + void merge(InstrProfRecord &Other, uint64_t Weight = 1); /// Scale up profile counts (including value profile data) by /// \p Weight. - instrprof_error scale(uint64_t Weight); + void scale(uint64_t Weight); /// Sort value profile data (per site) by count. void sortValueData() { @@ -424,6 +632,9 @@ struct InstrProfRecord { getValueSitesForKind(Kind).clear(); } + /// Get the error contained within the record's soft error counter. + Error takeError() { return SIPE.takeError(); } + private: std::vector<InstrProfValueSiteRecord> IndirectCallSites; const std::vector<InstrProfValueSiteRecord> & @@ -450,10 +661,10 @@ private: // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. - instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, - uint64_t Weight); + void mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, + uint64_t Weight); // Scale up value profile data count. - instrprof_error scaleValueProfData(uint32_t ValueKind, uint64_t Weight); + void scaleValueProfData(uint32_t ValueKind, uint64_t Weight); }; uint32_t InstrProfRecord::getNumValueKinds() const { @@ -482,29 +693,35 @@ uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, return getValueSitesForKind(ValueKind)[Site].ValueData.size(); } -std::unique_ptr<InstrProfValueData[]> InstrProfRecord::getValueForSite( - uint32_t ValueKind, uint32_t Site, - uint64_t (*ValueMapper)(uint32_t, uint64_t)) const { +std::unique_ptr<InstrProfValueData[]> +InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, + uint64_t *TotalC) const { + uint64_t Dummy; + uint64_t &TotalCount = (TotalC == 0 ? Dummy : *TotalC); uint32_t N = getNumValueDataForSite(ValueKind, Site); - if (N == 0) + if (N == 0) { + TotalCount = 0; return std::unique_ptr<InstrProfValueData[]>(nullptr); + } auto VD = llvm::make_unique<InstrProfValueData[]>(N); - getValueForSite(VD.get(), ValueKind, Site, ValueMapper); + TotalCount = getValueForSite(VD.get(), ValueKind, Site); return VD; } -void InstrProfRecord::getValueForSite(InstrProfValueData Dest[], - uint32_t ValueKind, uint32_t Site, - uint64_t (*ValueMapper)(uint32_t, - uint64_t)) const { +uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], + uint32_t ValueKind, + uint32_t Site) const { uint32_t I = 0; + uint64_t TotalCount = 0; for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { - Dest[I].Value = ValueMapper ? ValueMapper(ValueKind, V.Value) : V.Value; + Dest[I].Value = V.Value; Dest[I].Count = V.Count; + TotalCount = SaturatingAdd(TotalCount, V.Count); I++; } + return TotalCount; } void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { @@ -532,27 +749,6 @@ void InstrProfValueSiteRecord::sortByCount() { ValueData.resize(max_s); } -/* -* Initialize the record for runtime value profile data. -* Return 0 if the initialization is successful, otherwise -* return 1. -*/ -int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, - const uint16_t *NumValueSites, - ValueProfNode **Nodes); - -/* Release memory allocated for the runtime record. */ -void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); - -/* Return the size of ValueProfData structure that can be used to store - the value profile data collected at runtime. */ -uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); - -/* Return a ValueProfData instance that stores the data collected at runtime. */ -ValueProfData * -serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, - ValueProfData *Dst); - namespace IndexedInstrProf { enum class HashT : uint32_t { @@ -561,27 +757,38 @@ enum class HashT : uint32_t { Last = MD5 }; -static inline uint64_t MD5Hash(StringRef Str) { - MD5 Hash; - Hash.update(Str); - llvm::MD5::MD5Result Result; - Hash.final(Result); - // Return the least significant 8 bytes. Our MD5 implementation returns the - // result in little endian, so we may need to swap bytes. - using namespace llvm::support; - return endian::read<uint64_t, little, unaligned>(Result); -} - inline uint64_t ComputeHash(HashT Type, StringRef K) { switch (Type) { case HashT::MD5: - return IndexedInstrProf::MD5Hash(K); + return MD5Hash(K); } llvm_unreachable("Unhandled hash type"); } const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81" -const uint64_t Version = INSTR_PROF_INDEX_VERSION; + +enum ProfVersion { + // Version 1 is the first version. In this version, the value of + // a key/value pair can only include profile data of a single function. + // Due to this restriction, the number of block counters for a given + // function is not recorded but derived from the length of the value. + Version1 = 1, + // The version 2 format supports recording profile data of multiple + // functions which share the same key in one value field. To support this, + // the number block counters is recorded as an uint64_t field right after the + // function structural hash. + Version2 = 2, + // Version 3 supports value profile data. The value profile data is expected + // to follow the block counter profile data. + Version3 = 3, + // In this version, profile summary data \c IndexedInstrProf::Summary is + // stored after the profile header. + Version4 = 4, + // The current version is 4. + CurrentVersion = INSTR_PROF_INDEX_VERSION +}; +const uint64_t Version = ProfVersion::CurrentVersion; + const HashT HashType = HashT::MD5; inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } @@ -591,15 +798,104 @@ inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } struct Header { uint64_t Magic; uint64_t Version; - uint64_t MaxFunctionCount; + uint64_t Unused; // Becomes unused since version 4 uint64_t HashType; uint64_t HashOffset; }; +// Profile summary data recorded in the profile data file in indexed +// format. It is introduced in version 4. The summary data follows +// right after the profile file header. +struct Summary { + + struct Entry { + uint64_t Cutoff; ///< The required percentile of total execution count. + uint64_t + MinBlockCount; ///< The minimum execution count for this percentile. + uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count. + }; + // The field kind enumerator to assigned value mapping should remain + // unchanged when a new kind is added or an old kind gets deleted in + // the future. + enum SummaryFieldKind { + /// The total number of functions instrumented. + TotalNumFunctions = 0, + /// Total number of instrumented blocks/edges. + TotalNumBlocks = 1, + /// The maximal execution count among all functions. + /// This field does not exist for profile data from IR based + /// instrumentation. + MaxFunctionCount = 2, + /// Max block count of the program. + MaxBlockCount = 3, + /// Max internal block count of the program (excluding entry blocks). + MaxInternalBlockCount = 4, + /// The sum of all instrumented block counts. + TotalBlockCount = 5, + NumKinds = TotalBlockCount + 1 + }; + + // The number of summmary fields following the summary header. + uint64_t NumSummaryFields; + // The number of Cutoff Entries (Summary::Entry) following summary fields. + uint64_t NumCutoffEntries; + + static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) { + return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) + + NumSumFields * sizeof(uint64_t); + } + + const uint64_t *getSummaryDataBase() const { + return reinterpret_cast<const uint64_t *>(this + 1); + } + uint64_t *getSummaryDataBase() { + return reinterpret_cast<uint64_t *>(this + 1); + } + const Entry *getCutoffEntryBase() const { + return reinterpret_cast<const Entry *>( + &getSummaryDataBase()[NumSummaryFields]); + } + Entry *getCutoffEntryBase() { + return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]); + } + + uint64_t get(SummaryFieldKind K) const { + return getSummaryDataBase()[K]; + } + + void set(SummaryFieldKind K, uint64_t V) { + getSummaryDataBase()[K] = V; + } + + const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; } + void setEntry(uint32_t I, const ProfileSummaryEntry &E) { + Entry &ER = getCutoffEntryBase()[I]; + ER.Cutoff = E.Cutoff; + ER.MinBlockCount = E.MinCount; + ER.NumBlocks = E.NumCounts; + } + + Summary(uint32_t Size) { memset(this, 0, Size); } + void operator delete(void *ptr) { ::operator delete(ptr); } + + Summary() = delete; +}; + +inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) { + return std::unique_ptr<Summary>(new (::operator new(TotalSize)) + Summary(TotalSize)); +} } // end namespace IndexedInstrProf namespace RawInstrProf { +// Version 1: First version +// Version 2: Added value profile data section. Per-function control data +// struct has more fields to describe value profile information. +// Version 3: Compressed name section support. Function PGO name reference +// from control data struct is changed from raw pointer to Name's MD5 value. +// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the +// raw header. const uint64_t Version = INSTR_PROF_RAW_VERSION; template <class IntPtrT> inline uint64_t getMagic(); @@ -630,13 +926,8 @@ struct Header { #include "llvm/ProfileData/InstrProfData.inc" }; -} // end namespace RawInstrProf +} // end namespace RawInstrProf } // end namespace llvm -namespace std { -template <> -struct is_error_code_enum<llvm::instrprof_error> : std::true_type {}; -} - -#endif // LLVM_PROFILEDATA_INSTRPROF_H_ +#endif // LLVM_PROFILEDATA_INSTRPROF_H diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc index 33c7d94aea2a2..4138e18fa22f9 100644 --- a/include/llvm/ProfileData/InstrProfData.inc +++ b/include/llvm/ProfileData/InstrProfData.inc @@ -57,6 +57,12 @@ * \*===----------------------------------------------------------------------===*/ +/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in + * the compiler runtime. */ +#ifndef INSTR_PROF_VISIBILITY +#define INSTR_PROF_VISIBILITY +#endif + /* INSTR_PROF_DATA start. */ /* Definition of member fields of the per-function control structure. */ #ifndef INSTR_PROF_DATA @@ -64,29 +70,57 @@ #else #define INSTR_PROF_DATA_DEFINED #endif - -INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ - NamePtr->getType()->getPointerElementType()->getArrayNumElements())) -INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) +INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \ - ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx))) INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ ConstantExpr::getBitCast(CounterPtr, \ llvm::Type::getInt64PtrTy(Ctx))) +/* This is used to map function pointers for the indirect call targets to + * function name hashes during the conversion from raw to merged profile + * data. + */ INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ FunctionAddr) INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ - ConstantPointerNull::get(Int8PtrTy)) + ValuesPtrExpr) +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ + +/* This is an internal data structure used by value profiler. It + * is defined here to allow serialization code sharing by LLVM + * to be used in unit test. + * + * typedef struct ValueProfNode { + * // InstrProfValueData VData; + * uint64_t Value; + * uint64_t Count; + * struct ValueProfNode *Next; + * } ValueProfNode; + */ +/* INSTR_PROF_VALUE_NODE start. */ +#ifndef INSTR_PROF_VALUE_NODE +#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ + ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) +INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ + ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) +INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ + ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) +#undef INSTR_PROF_VALUE_NODE +/* INSTR_PROF_VALUE_NODE end. */ + /* INSTR_PROF_RAW_HEADER start */ /* Definition of member fields of the raw profile header data structure. */ #ifndef INSTR_PROF_RAW_HEADER @@ -102,8 +136,6 @@ INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) -INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize) -INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ @@ -132,6 +164,15 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) #else #define INSTR_PROF_DATA_DEFINED #endif +/* For indirect function call value profiling, the addresses of the target + * functions are profiled by the instrumented code. The target addresses are + * written in the raw profile data and converted to target function name's MD5 + * hash by the profile reader during deserialization. Typically, this happens + * when the the raw profile data is read during profile merging. + * + * For this remapping the ProfData is used. ProfData contains both the function + * name hash and the function address. + */ VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) /* These two kinds must be the last to be * declared. This is to make sure the string @@ -153,12 +194,18 @@ VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) #else #define INSTR_PROF_DATA_DEFINED #endif +#ifdef COVMAP_V1 COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ llvm::Type::getInt8PtrTy(Ctx))) COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ - llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ NameValue.size())) +#else +COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ + llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + llvm::IndexedInstrProf::ComputeHash(NameValue))) +#endif COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ CoverageMapping.size())) @@ -182,7 +229,7 @@ COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) COVMAP_HEADER(uint32_t, Int32Ty, Version, \ - llvm::ConstantInt::get(Int32Ty, CoverageMappingCurrentVersion)) + llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) #undef COVMAP_HEADER /* COVMAP_HEADER end. */ @@ -281,16 +328,15 @@ typedef struct ValueProfData { static std::unique_ptr<ValueProfData> serializeFrom(const InstrProfRecord &Record); /*! - * Check the integrity of the record. Return the error code when - * an error is detected, otherwise return instrprof_error::success. + * Check the integrity of the record. */ - instrprof_error checkIntegrity(); + Error checkIntegrity(); /*! * Return a pointer to \c ValueProfileData instance ready to be read. * All data in the instance are properly byte swapped. The input * data is assumed to be in little endian order. */ - static ErrorOr<std::unique_ptr<ValueProfData>> + static Expected<std::unique_ptr<ValueProfData>> getValueProfData(const unsigned char *SrcBuffer, const unsigned char *const SrcBufferEnd, support::endianness SrcDataEndianness); @@ -343,46 +389,18 @@ typedef struct ValueProfRecordClosure { */ uint64_t (*RemapValueData)(uint32_t, uint64_t Value); void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, - uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)); + uint32_t S); ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); } ValueProfRecordClosure; -/* - * A wrapper struct that represents value profile runtime data. - * Like InstrProfRecord class which is used by profiling host tools, - * ValueProfRuntimeRecord also implements the abstract intefaces defined in - * ValueProfRecordClosure so that the runtime data can be serialized using - * shared C implementation. In this structure, NumValueSites and Nodes - * members are the primary fields while other fields hold the derived - * information for fast implementation of closure interfaces. - */ -typedef struct ValueProfRuntimeRecord { - /* Number of sites for each value profile kind. */ - const uint16_t *NumValueSites; - /* An array of linked-list headers. The size of of the array is the - * total number of value profile sites : sum(NumValueSites[*])). Each - * linked-list stores the values profiled for a value profile site. */ - ValueProfNode **Nodes; - - /* Total number of value profile kinds which have at least one - * value profile sites. */ - uint32_t NumValueKinds; - /* An array recording the number of values tracked at each site. - * The size of the array is TotalNumValueSites. */ - uint8_t *SiteCountArray[IPVK_Last + 1]; - ValueProfNode **NodesKind[IPVK_Last + 1]; -} ValueProfRuntimeRecord; - -/* Forward declarations of C interfaces. */ -int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, - const uint16_t *NumValueSites, - ValueProfNode **Nodes); -void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); -uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); -ValueProfData * -serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, - ValueProfData *Dst); -uint32_t getNumValueKindsRT(const void *R); +INSTR_PROF_VISIBILITY ValueProfRecord * +getFirstValueProfRecord(ValueProfData *VPD); +INSTR_PROF_VISIBILITY ValueProfRecord * +getValueProfRecordNext(ValueProfRecord *VPR); +INSTR_PROF_VISIBILITY InstrProfValueData * +getValueProfRecordValueData(ValueProfRecord *VPR); +INSTR_PROF_VISIBILITY uint32_t +getValueProfRecordHeaderSize(uint32_t NumValueSites); #undef INSTR_PROF_VALUE_PROF_DATA #endif /* INSTR_PROF_VALUE_PROF_DATA */ @@ -392,8 +410,10 @@ uint32_t getNumValueKindsRT(const void *R); #define INSTR_PROF_DATA_DEFINED #ifdef __cplusplus #define INSTR_PROF_INLINE inline +#define INSTR_PROF_NULLPTR nullptr #else #define INSTR_PROF_INLINE +#define INSTR_PROF_NULLPTR NULL #endif #ifndef offsetof @@ -404,7 +424,7 @@ uint32_t getNumValueKindsRT(const void *R); * \brief Return the \c ValueProfRecord header size including the * padding bytes. */ -INSTR_PROF_INLINE +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + sizeof(uint8_t) * NumValueSites; @@ -417,7 +437,7 @@ uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { * \brief Return the total size of the value profile record including the * header and the value data. */ -INSTR_PROF_INLINE +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) { return getValueProfRecordHeaderSize(NumValueSites) + @@ -427,7 +447,7 @@ uint32_t getValueProfRecordSize(uint32_t NumValueSites, /*! * \brief Return the pointer to the start of value data array. */ -INSTR_PROF_INLINE +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( This->NumValueSites)); @@ -436,7 +456,7 @@ InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { /*! * \brief Return the total number of value data for \c This record. */ -INSTR_PROF_INLINE +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { uint32_t NumValueData = 0; uint32_t I; @@ -448,7 +468,7 @@ uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { /*! * \brief Use this method to advance to the next \c This \c ValueProfRecord. */ -INSTR_PROF_INLINE +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { uint32_t NumValueData = getValueProfRecordNumValueData(This); return (ValueProfRecord *)((char *)This + @@ -459,7 +479,7 @@ ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { /*! * \brief Return the first \c ValueProfRecord instance. */ -INSTR_PROF_INLINE +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); } @@ -470,13 +490,11 @@ ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { * Return the total size in bytes of the on-disk value profile data * given the data stored in Record. */ -uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { +INSTR_PROF_VISIBILITY uint32_t +getValueProfDataSize(ValueProfRecordClosure *Closure) { uint32_t Kind; uint32_t TotalSize = sizeof(ValueProfData); const void *Record = Closure->Record; - uint32_t NumValueKinds = Closure->GetNumValueKinds(Record); - if (NumValueKinds == 0) - return TotalSize; for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); @@ -492,9 +510,10 @@ uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { * Extract value profile data of a function for the profile kind \c ValueKind * from the \c Closure and serialize the data into \c This record instance. */ -void serializeValueProfRecordFrom(ValueProfRecord *This, - ValueProfRecordClosure *Closure, - uint32_t ValueKind, uint32_t NumValueSites) { +INSTR_PROF_VISIBILITY void +serializeValueProfRecordFrom(ValueProfRecord *This, + ValueProfRecordClosure *Closure, + uint32_t ValueKind, uint32_t NumValueSites) { uint32_t S; const void *Record = Closure->Record; This->Kind = ValueKind; @@ -504,8 +523,7 @@ void serializeValueProfRecordFrom(ValueProfRecord *This, for (S = 0; S < NumValueSites; S++) { uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); This->SiteCountArray[S] = ND; - Closure->GetValueForSite(Record, DstVD, ValueKind, S, - Closure->RemapValueData); + Closure->GetValueForSite(Record, DstVD, ValueKind, S); DstVD += ND; } } @@ -513,12 +531,16 @@ void serializeValueProfRecordFrom(ValueProfRecord *This, /*! * Extract value profile data of a function from the \c Closure * and serialize the data into \c DstData if it is not NULL or heap - * memory allocated by the \c Closure's allocator method. + * memory allocated by the \c Closure's allocator method. If \c + * DstData is not null, the caller is expected to set the TotalSize + * in DstData. */ -ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, - ValueProfData *DstData) { +INSTR_PROF_VISIBILITY ValueProfData * +serializeValueProfDataFrom(ValueProfRecordClosure *Closure, + ValueProfData *DstData) { uint32_t Kind; - uint32_t TotalSize = getValueProfDataSize(Closure); + uint32_t TotalSize = + DstData ? DstData->TotalSize : getValueProfDataSize(Closure); ValueProfData *VPD = DstData ? DstData : Closure->AllocValueProfData(TotalSize); @@ -536,144 +558,15 @@ ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, return VPD; } -/* - * The value profiler runtime library stores the value profile data - * for a given function in \c NumValueSites and \c Nodes structures. - * \c ValueProfRuntimeRecord class is used to encapsulate the runtime - * profile data and provides fast interfaces to retrieve the profile - * information. This interface is used to initialize the runtime record - * and pre-compute the information needed for efficient implementation - * of callbacks required by ValueProfRecordClosure class. - */ -int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, - const uint16_t *NumValueSites, - ValueProfNode **Nodes) { - unsigned I, J, S = 0, NumValueKinds = 0; - RuntimeRecord->NumValueSites = NumValueSites; - RuntimeRecord->Nodes = Nodes; - for (I = 0; I <= IPVK_Last; I++) { - uint16_t N = NumValueSites[I]; - if (!N) { - RuntimeRecord->SiteCountArray[I] = 0; - continue; - } - NumValueKinds++; - RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1); - if (!RuntimeRecord->SiteCountArray[I]) - return 1; - RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : NULL; - for (J = 0; J < N; J++) { - /* Compute value count for each site. */ - uint32_t C = 0; - ValueProfNode *Site = Nodes ? RuntimeRecord->NodesKind[I][J] : NULL; - while (Site) { - C++; - Site = Site->Next; - } - if (C > UCHAR_MAX) - C = UCHAR_MAX; - RuntimeRecord->SiteCountArray[I][J] = C; - } - S += N; - } - RuntimeRecord->NumValueKinds = NumValueKinds; - return 0; -} - -void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) { - unsigned I; - for (I = 0; I <= IPVK_Last; I++) { - if (RuntimeRecord->SiteCountArray[I]) - free(RuntimeRecord->SiteCountArray[I]); - } -} - -/* ValueProfRecordClosure Interface implementation for - * ValueProfDataRuntimeRecord. */ -uint32_t getNumValueKindsRT(const void *R) { - return ((const ValueProfRuntimeRecord *)R)->NumValueKinds; -} - -uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { - return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK]; -} - -uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) { - const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; - return Record->SiteCountArray[VK][S]; -} - -uint32_t getNumValueDataRT(const void *R, uint32_t VK) { - unsigned I, S = 0; - const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; - if (Record->SiteCountArray[VK] == 0) - return 0; - for (I = 0; I < Record->NumValueSites[VK]; I++) - S += Record->SiteCountArray[VK][I]; - return S; -} - -void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK, - uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) { - unsigned I, N = 0; - const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; - N = getNumValueDataForSiteRT(R, VK, S); - if (N == 0) - return; - ValueProfNode *VNode = Record->NodesKind[VK][S]; - for (I = 0; I < N; I++) { - Dst[I] = VNode->VData; - VNode = VNode->Next; - } -} - -ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) { - return (ValueProfData *)calloc(TotalSizeInBytes, 1); -} - -static ValueProfRecordClosure RTRecordClosure = {0, - getNumValueKindsRT, - getNumValueSitesRT, - getNumValueDataRT, - getNumValueDataForSiteRT, - 0, - getValueForSiteRT, - allocValueProfDataRT}; - -/* - * Return the size of ValueProfData structure to store data - * recorded in the runtime record. - */ -uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { - RTRecordClosure.Record = Record; - return getValueProfDataSize(&RTRecordClosure); -} - -/* - * Return a ValueProfData instance that stores the data collected - * from runtime. If \c DstData is provided by the caller, the value - * profile data will be store in *DstData and DstData is returned, - * otherwise the method will allocate space for the value data and - * return pointer to the newly allocated space. - */ -ValueProfData * -serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, - ValueProfData *DstData) { - RTRecordClosure.Record = Record; - return serializeValueProfDataFrom(&RTRecordClosure, DstData); -} - - #undef INSTR_PROF_COMMON_API_IMPL #endif /* INSTR_PROF_COMMON_API_IMPL */ /*============================================================================*/ - #ifndef INSTR_PROF_DATA_DEFINED -#ifndef INSTR_PROF_DATA_INC_ -#define INSTR_PROF_DATA_INC_ +#ifndef INSTR_PROF_DATA_INC +#define INSTR_PROF_DATA_INC /* Helper macros. */ #define INSTR_PROF_SIMPLE_QUOTE(x) #x @@ -695,23 +588,33 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 -/* Raw profile format version. */ -#define INSTR_PROF_RAW_VERSION 2 -#define INSTR_PROF_INDEX_VERSION 3 -#define INSTR_PROF_COVMAP_VERSION 0 +/* Raw profile format version (start from 1). */ +#define INSTR_PROF_RAW_VERSION 4 +/* Indexed profile format version (start from 1). */ +#define INSTR_PROF_INDEX_VERSION 4 +/* Coverage mapping format vresion (start from 0). */ +#define INSTR_PROF_COVMAP_VERSION 1 -/* Profile version is always of type uint_64_t. Reserve the upper 8 bits in the +/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. -*/ + */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) +#define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define IR_LEVEL_PROF_VERSION_VAR __llvm_profile_raw_version /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names #define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts +/* Array of pointers. Each pointer points to a list + * of value nodes associated with one value site. + */ +#define INSTR_PROF_VALS_SECT_NAME __llvm_prf_vals +/* Value profile nodes section. */ +#define INSTR_PROF_VNODES_SECT_NAME __llvm_prf_vnds #define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap #define INSTR_PROF_DATA_SECT_NAME_STR \ @@ -722,6 +625,10 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) #define INSTR_PROF_COVMAP_SECT_NAME_STR \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) +#define INSTR_PROF_VALS_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME) +#define INSTR_PROF_VNODES_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -751,16 +658,7 @@ typedef struct InstrProfValueData { uint64_t Count; } InstrProfValueData; -/* This is an internal data structure used by value profiler. It - * is defined here to allow serialization code sharing by LLVM - * to be used in unit test. - */ -typedef struct ValueProfNode { - InstrProfValueData VData; - struct ValueProfNode *Next; -} ValueProfNode; - -#endif /* INSTR_PROF_DATA_INC_ */ +#endif /* INSTR_PROF_DATA_INC */ #else #undef INSTR_PROF_DATA_DEFINED diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h index fed3e693e7a05..65b11f61d10bc 100644 --- a/include/llvm/ProfileData/InstrProfReader.h +++ b/include/llvm/ProfileData/InstrProfReader.h @@ -19,7 +19,6 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/EndianStream.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/OnDiskHashTable.h" @@ -51,19 +50,20 @@ public: /// Base class and interface for reading profiling data of any known instrprof /// format. Provides an iterator over InstrProfRecords. class InstrProfReader { - std::error_code LastError; + instrprof_error LastError; public: InstrProfReader() : LastError(instrprof_error::success), Symtab() {} virtual ~InstrProfReader() {} /// Read the header. Required before reading first record. - virtual std::error_code readHeader() = 0; + virtual Error readHeader() = 0; /// Read a single record. - virtual std::error_code readNextRecord(InstrProfRecord &Record) = 0; + virtual Error readNextRecord(InstrProfRecord &Record) = 0; /// Iterator over profile data. InstrProfIterator begin() { return InstrProfIterator(this); } InstrProfIterator end() { return InstrProfIterator(); } + virtual bool isIRLevelProfile() const = 0; /// Return the PGO symtab. There are three different readers: /// Raw, Text, and Indexed profile readers. The first two types @@ -79,28 +79,35 @@ public: protected: std::unique_ptr<InstrProfSymtab> Symtab; - /// Set the current std::error_code and return same. - std::error_code error(std::error_code EC) { - LastError = EC; - return EC; + /// Set the current error and return same. + Error error(instrprof_error Err) { + LastError = Err; + if (Err == instrprof_error::success) + return Error::success(); + return make_error<InstrProfError>(Err); } + Error error(Error E) { return error(InstrProfError::take(std::move(E))); } - /// Clear the current error code and return a successful one. - std::error_code success() { return error(instrprof_error::success); } + /// Clear the current error and return a successful one. + Error success() { return error(instrprof_error::success); } public: /// Return true if the reader has finished reading the profile data. bool isEOF() { return LastError == instrprof_error::eof; } /// Return true if the reader encountered an error reading profiling data. - bool hasError() { return LastError && !isEOF(); } - /// Get the current error code. - std::error_code getError() { return LastError; } + bool hasError() { return LastError != instrprof_error::success && !isEOF(); } + /// Get the current error. + Error getError() { + if (hasError()) + return make_error<InstrProfError>(LastError); + return Error::success(); + } /// Factory method to create an appropriately typed reader for the given /// instrprof file. - static ErrorOr<std::unique_ptr<InstrProfReader>> create(std::string Path); + static Expected<std::unique_ptr<InstrProfReader>> create(const Twine &Path); - static ErrorOr<std::unique_ptr<InstrProfReader>> + static Expected<std::unique_ptr<InstrProfReader>> create(std::unique_ptr<MemoryBuffer> Buffer); }; @@ -118,22 +125,26 @@ private: std::unique_ptr<MemoryBuffer> DataBuffer; /// Iterator over the profile data. line_iterator Line; + bool IsIRLevelProfile; TextInstrProfReader(const TextInstrProfReader &) = delete; TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; - std::error_code readValueProfileData(InstrProfRecord &Record); + Error readValueProfileData(InstrProfRecord &Record); public: TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_) - : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} + : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#'), + IsIRLevelProfile(false) {} /// Return true if the given buffer is in text instrprof format. static bool hasFormat(const MemoryBuffer &Buffer); + bool isIRLevelProfile() const override { return IsIRLevelProfile; } + /// Read the header. - std::error_code readHeader() override; + Error readHeader() override; /// Read a single record. - std::error_code readNextRecord(InstrProfRecord &Record) override; + Error readNextRecord(InstrProfRecord &Record) override; InstrProfSymtab &getSymtab() override { assert(Symtab.get()); @@ -154,14 +165,20 @@ private: /// The profile data file contents. std::unique_ptr<MemoryBuffer> DataBuffer; bool ShouldSwapBytes; + // The value of the version field of the raw profile data header. The lower 56 + // bits specifies the format version and the most significant 8 bits specify + // the variant types of the profile. + uint64_t Version; uint64_t CountersDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData<IntPtrT> *Data; const RawInstrProf::ProfileData<IntPtrT> *DataEnd; const uint64_t *CountersStart; const char *NamesStart; + uint64_t NamesSize; + // After value profile is all read, this pointer points to + // the header of next profile data (if exists) const uint8_t *ValueDataStart; - const char *ProfileEnd; uint32_t ValueKindLast; uint32_t CurValueDataSize; @@ -174,8 +191,11 @@ public: : DataBuffer(std::move(DataBuffer)) { } static bool hasFormat(const MemoryBuffer &DataBuffer); - std::error_code readHeader() override; - std::error_code readNextRecord(InstrProfRecord &Record) override; + Error readHeader() override; + Error readNextRecord(InstrProfRecord &Record) override; + bool isIRLevelProfile() const override { + return (Version & VARIANT_MASK_IR_PROF) != 0; + } InstrProfSymtab &getSymtab() override { assert(Symtab.get()); @@ -183,9 +203,9 @@ public: } private: - void createSymtab(InstrProfSymtab &Symtab); - std::error_code readNextHeader(const char *CurrentPos); - std::error_code readHeader(const RawInstrProf::Header &Header); + Error createSymtab(InstrProfSymtab &Symtab); + Error readNextHeader(const char *CurrentPos); + Error readHeader(const RawInstrProf::Header &Header); template <class IntT> IntT swap(IntT Int) const { return ShouldSwapBytes ? sys::getSwappedBytes(Int) : Int; } @@ -202,23 +222,26 @@ private: inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) { return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); } - std::error_code readName(InstrProfRecord &Record); - std::error_code readFuncHash(InstrProfRecord &Record); - std::error_code readRawCounts(InstrProfRecord &Record); - std::error_code readValueProfilingData(InstrProfRecord &Record); + Error readName(InstrProfRecord &Record); + Error readFuncHash(InstrProfRecord &Record); + Error readRawCounts(InstrProfRecord &Record); + Error readValueProfilingData(InstrProfRecord &Record); bool atEnd() const { return Data == DataEnd; } void advanceData() { Data++; ValueDataStart += CurValueDataSize; } + const char *getNextHeaderPos() const { + assert(atEnd()); + return (const char *)ValueDataStart; + } const uint64_t *getCounter(IntPtrT CounterPtr) const { ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); return CountersStart + Offset; } - const char *getName(IntPtrT NamePtr) const { - ptrdiff_t Offset = (swap(NamePtr) - NamesDelta) / sizeof(char); - return NamesStart + Offset; + StringRef getName(uint64_t NameRef) const { + return Symtab->getFuncName(swap(NameRef)); } }; @@ -283,15 +306,16 @@ public: struct InstrProfReaderIndexBase { // Read all the profile records with the same key pointed to the current // iterator. - virtual std::error_code getRecords(ArrayRef<InstrProfRecord> &Data) = 0; + virtual Error getRecords(ArrayRef<InstrProfRecord> &Data) = 0; // Read all the profile records with the key equal to FuncName - virtual std::error_code getRecords(StringRef FuncName, + virtual Error getRecords(StringRef FuncName, ArrayRef<InstrProfRecord> &Data) = 0; virtual void advanceToNextKey() = 0; virtual bool atEnd() const = 0; virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; virtual ~InstrProfReaderIndexBase() {} virtual uint64_t getVersion() const = 0; + virtual bool isIRLevelProfile() const = 0; virtual void populateSymtab(InstrProfSymtab &) = 0; }; @@ -312,9 +336,9 @@ public: const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version); - std::error_code getRecords(ArrayRef<InstrProfRecord> &Data) override; - std::error_code getRecords(StringRef FuncName, - ArrayRef<InstrProfRecord> &Data) override; + Error getRecords(ArrayRef<InstrProfRecord> &Data) override; + Error getRecords(StringRef FuncName, + ArrayRef<InstrProfRecord> &Data) override; void advanceToNextKey() override { RecordIterator++; } bool atEnd() const override { return RecordIterator == HashTable->data_end(); @@ -323,7 +347,10 @@ public: HashTable->getInfoObj().setValueProfDataEndianness(Endianness); } ~InstrProfReaderIndex() override {} - uint64_t getVersion() const override { return FormatVersion; } + uint64_t getVersion() const override { return GET_VERSION(FormatVersion); } + bool isIRLevelProfile() const override { + return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; + } void populateSymtab(InstrProfSymtab &Symtab) override { Symtab.create(HashTable->keys()); } @@ -336,14 +363,21 @@ private: std::unique_ptr<MemoryBuffer> DataBuffer; /// The index into the profile data. std::unique_ptr<InstrProfReaderIndexBase> Index; - /// The maximal execution count among all functions. - uint64_t MaxFunctionCount; + /// Profile summary data. + std::unique_ptr<ProfileSummary> Summary; IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; + // Read the profile summary. Return a pointer pointing to one byte past the + // end of the summary data if it exists or the input \c Cur. + const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version, + const unsigned char *Cur); + public: + /// Return the profile version. uint64_t getVersion() const { return Index->getVersion(); } + bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } IndexedInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer) : DataBuffer(std::move(DataBuffer)), Index(nullptr) {} @@ -351,27 +385,27 @@ public: static bool hasFormat(const MemoryBuffer &DataBuffer); /// Read the file header. - std::error_code readHeader() override; + Error readHeader() override; /// Read a single record. - std::error_code readNextRecord(InstrProfRecord &Record) override; + Error readNextRecord(InstrProfRecord &Record) override; /// Return the pointer to InstrProfRecord associated with FuncName /// and FuncHash - ErrorOr<InstrProfRecord> getInstrProfRecord(StringRef FuncName, - uint64_t FuncHash); + Expected<InstrProfRecord> getInstrProfRecord(StringRef FuncName, + uint64_t FuncHash); /// Fill Counts with the profile data for the given function name. - std::error_code getFunctionCounts(StringRef FuncName, uint64_t FuncHash, - std::vector<uint64_t> &Counts); + Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, + std::vector<uint64_t> &Counts); /// Return the maximum of all known function counts. - uint64_t getMaximumFunctionCount() { return MaxFunctionCount; } + uint64_t getMaximumFunctionCount() { return Summary->getMaxFunctionCount(); } /// Factory method to create an indexed reader. - static ErrorOr<std::unique_ptr<IndexedInstrProfReader>> - create(std::string Path); + static Expected<std::unique_ptr<IndexedInstrProfReader>> + create(const Twine &Path); - static ErrorOr<std::unique_ptr<IndexedInstrProfReader>> + static Expected<std::unique_ptr<IndexedInstrProfReader>> create(std::unique_ptr<MemoryBuffer> Buffer); // Used for testing purpose only. @@ -383,6 +417,7 @@ public: // to be used by llvm-profdata (for dumping). Avoid using this when // the client is the compiler. InstrProfSymtab &getSymtab() override; + ProfileSummary &getSummary() { return *(Summary.get()); } }; } // end namespace llvm diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h index e7f53de051c3c..7d292731cccb8 100644 --- a/include/llvm/ProfileData/InstrProfWriter.h +++ b/include/llvm/ProfileData/InstrProfWriter.h @@ -24,21 +24,29 @@ namespace llvm { /// Writer for instrumentation based profile data. +class ProfOStream; +class InstrProfRecordWriterTrait; + class InstrProfWriter { public: typedef SmallDenseMap<uint64_t, InstrProfRecord, 1> ProfilingData; + enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; private: + bool Sparse; StringMap<ProfilingData> FunctionData; - uint64_t MaxFunctionCount; + ProfKind ProfileKind; + // Use raw pointer here for the incomplete type object. + InstrProfRecordWriterTrait *InfoObj; public: - InstrProfWriter() : MaxFunctionCount(0) {} + InstrProfWriter(bool Sparse = false); + ~InstrProfWriter(); /// Add function counts for the given function. If there are already counts /// for this function and the hash and number of counts match, each counter is /// summed. Optionally scale counts by \p Weight. - std::error_code addRecord(InstrProfRecord &&I, uint64_t Weight = 1); + Error addRecord(InstrProfRecord &&I, uint64_t Weight = 1); /// Write the profile to \c OS void write(raw_fd_ostream &OS); /// Write the profile in text format to \c OS @@ -49,11 +57,25 @@ public: /// Write the profile, returning the raw data. For testing. std::unique_ptr<MemoryBuffer> writeBuffer(); + /// Set the ProfileKind. Report error if mixing FE and IR level profiles. + Error setIsIRLevelProfile(bool IsIRLevel) { + if (ProfileKind == PF_Unknown) { + ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE; + return Error::success(); + } + return (IsIRLevel == (ProfileKind == PF_IRLevel)) + ? Error::success() + : make_error<InstrProfError>( + instrprof_error::unsupported_version); + } + // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); + void setOutputSparse(bool Sparse); private: - std::pair<uint64_t, uint64_t> writeImpl(raw_ostream &OS); + bool shouldEncodeData(const ProfilingData &PD); + void writeImpl(ProfOStream &OS); }; } // end namespace llvm diff --git a/include/llvm/ProfileData/ProfileCommon.h b/include/llvm/ProfileData/ProfileCommon.h new file mode 100644 index 0000000000000..ecb228ca59c47 --- /dev/null +++ b/include/llvm/ProfileData/ProfileCommon.h @@ -0,0 +1,102 @@ +//===-- ProfileCommon.h - Common profiling APIs. ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains data structures and functions common to both instrumented +// and sample profiling. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_PROFILE_COMMON_H +#define LLVM_PROFILEDATA_PROFILE_COMMON_H + +#include <cstdint> +#include <functional> +#include <map> +#include <utility> +#include <vector> + +#include "llvm/IR/ProfileSummary.h" +#include "llvm/Support/Error.h" +#include "llvm/ADT/ArrayRef.h" + +namespace llvm { +class Function; +namespace IndexedInstrProf { +struct Summary; +} +namespace sampleprof { +class FunctionSamples; +} +struct InstrProfRecord; +class LLVMContext; +class Metadata; +class MDTuple; +class MDNode; + +inline const char *getHotSectionPrefix() { return ".hot"; } +inline const char *getUnlikelySectionPrefix() { return ".unlikely"; } + +class ProfileSummaryBuilder { + +private: + // We keep track of the number of times a count (block count or samples) + // appears in the profile. The map is kept sorted in the descending order of + // counts. + std::map<uint64_t, uint32_t, std::greater<uint64_t>> CountFrequencies; + std::vector<uint32_t> DetailedSummaryCutoffs; + +protected: + SummaryEntryVector DetailedSummary; + ProfileSummaryBuilder(std::vector<uint32_t> Cutoffs) + : DetailedSummaryCutoffs(std::move(Cutoffs)), TotalCount(0), MaxCount(0), + MaxFunctionCount(0), NumCounts(0), NumFunctions(0) {} + inline void addCount(uint64_t Count); + ~ProfileSummaryBuilder() = default; + void computeDetailedSummary(); + uint64_t TotalCount, MaxCount, MaxFunctionCount; + uint32_t NumCounts, NumFunctions; + +public: + /// \brief A vector of useful cutoff values for detailed summary. + static const ArrayRef<uint32_t> DefaultCutoffs; +}; + +class InstrProfSummaryBuilder final : public ProfileSummaryBuilder { + uint64_t MaxInternalBlockCount; + inline void addEntryCount(uint64_t Count); + inline void addInternalCount(uint64_t Count); + +public: + InstrProfSummaryBuilder(std::vector<uint32_t> Cutoffs) + : ProfileSummaryBuilder(std::move(Cutoffs)), MaxInternalBlockCount(0) {} + void addRecord(const InstrProfRecord &); + std::unique_ptr<ProfileSummary> getSummary(); +}; + +class SampleProfileSummaryBuilder final : public ProfileSummaryBuilder { + +public: + void addRecord(const sampleprof::FunctionSamples &FS); + SampleProfileSummaryBuilder(std::vector<uint32_t> Cutoffs) + : ProfileSummaryBuilder(std::move(Cutoffs)) {} + std::unique_ptr<ProfileSummary> getSummary(); +}; + +// This is called when a count is seen in the profile. +void ProfileSummaryBuilder::addCount(uint64_t Count) { + TotalCount += Count; + if (Count > MaxCount) + MaxCount = Count; + NumCounts++; + CountFrequencies[Count]++; +} + + +} // end namespace llvm +#endif diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 6c39cf9458dcb..9fefefa627b1d 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -74,7 +74,7 @@ static inline uint64_t SPMagic() { uint64_t('2') << (64 - 56) | uint64_t(0xff); } -static inline uint64_t SPVersion() { return 102; } +static inline uint64_t SPVersion() { return 103; } /// Represents the relative location of an instruction. /// @@ -100,23 +100,6 @@ struct LineLocation { raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); -/// Represents the relative location of a callsite. -/// -/// Callsite locations are specified by the line offset from the -/// beginning of the function (marked by the line where the function -/// head is), the discriminator value within that line, and the callee -/// function name. -struct CallsiteLocation : public LineLocation { - CallsiteLocation(uint32_t L, uint32_t D, StringRef N) - : LineLocation(L, D), CalleeName(N) {} - void print(raw_ostream &OS) const; - void dump() const; - - StringRef CalleeName; -}; - -raw_ostream &operator<<(raw_ostream &OS, const CallsiteLocation &Loc); - /// Representation of a single sample record. /// /// A sample record is represented by a positive integer value, which @@ -188,7 +171,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample); typedef std::map<LineLocation, SampleRecord> BodySampleMap; class FunctionSamples; -typedef std::map<CallsiteLocation, FunctionSamples> CallsiteSampleMap; +typedef std::map<LineLocation, FunctionSamples> CallsiteSampleMap; /// Representation of the samples collected for a function. /// @@ -197,7 +180,7 @@ typedef std::map<CallsiteLocation, FunctionSamples> CallsiteSampleMap; /// within the body of the function. class FunctionSamples { public: - FunctionSamples() : TotalSamples(0), TotalHeadSamples(0) {} + FunctionSamples() : Name(), TotalSamples(0), TotalHeadSamples(0) {} void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const; void dump() const; sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) { @@ -221,8 +204,8 @@ public: } sampleprof_error addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator, - std::string FName, uint64_t Num, - uint64_t Weight = 1) { + const std::string &FName, + uint64_t Num, uint64_t Weight = 1) { return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget( FName, Num, Weight); } @@ -240,13 +223,12 @@ public: } /// Return the function samples at the given callsite location. - FunctionSamples &functionSamplesAt(const CallsiteLocation &Loc) { + FunctionSamples &functionSamplesAt(const LineLocation &Loc) { return CallsiteSamples[Loc]; } /// Return a pointer to function samples at the given callsite location. - const FunctionSamples * - findFunctionSamplesAt(const CallsiteLocation &Loc) const { + const FunctionSamples *findFunctionSamplesAt(const LineLocation &Loc) const { auto iter = CallsiteSamples.find(Loc); if (iter == CallsiteSamples.end()) { return nullptr; @@ -276,6 +258,7 @@ public: /// Optionally scale samples by \p Weight. sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) { sampleprof_error Result = sampleprof_error::success; + Name = Other.getName(); MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight)); MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight)); for (const auto &I : Other.getBodySamples()) { @@ -284,14 +267,23 @@ public: MergeResult(Result, BodySamples[Loc].merge(Rec, Weight)); } for (const auto &I : Other.getCallsiteSamples()) { - const CallsiteLocation &Loc = I.first; + const LineLocation &Loc = I.first; const FunctionSamples &Rec = I.second; MergeResult(Result, functionSamplesAt(Loc).merge(Rec, Weight)); } return Result; } + /// Set the name of the function. + void setName(StringRef FunctionName) { Name = FunctionName; } + + /// Return the function name. + const StringRef &getName() const { return Name; } + private: + /// Mangled name of the function. + StringRef Name; + /// Total number of samples collected inside this function. /// /// Samples are cumulative, they include all the samples collected diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index 6db0fbb0e7abf..bf86721709c7b 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -129,6 +129,30 @@ // VERSION (uint32_t) // File format version number computed by SPVersion() // +// SUMMARY +// TOTAL_COUNT (uint64_t) +// Total number of samples in the profile. +// MAX_COUNT (uint64_t) +// Maximum value of samples on a line. +// MAX_FUNCTION_COUNT (uint64_t) +// Maximum number of samples at function entry (head samples). +// NUM_COUNTS (uint64_t) +// Number of lines with samples. +// NUM_FUNCTIONS (uint64_t) +// Number of functions with samples. +// NUM_DETAILED_SUMMARY_ENTRIES (size_t) +// Number of entries in detailed summary +// DETAILED_SUMMARY +// A list of detailed summary entry. Each entry consists of +// CUTOFF (uint32_t) +// Required percentile of total sample count expressed as a fraction +// multiplied by 1000000. +// MIN_COUNT (uint64_t) +// The minimum number of samples required to reach the target +// CUTOFF. +// NUM_COUNTS (uint64_t) +// Number of samples to get to the desrired percentile. +// // NAME TABLE // SIZE (uint32_t) // Number of entries in the name table. @@ -190,6 +214,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -264,12 +289,15 @@ public: /// \brief Create a sample profile reader appropriate to the file format. static ErrorOr<std::unique_ptr<SampleProfileReader>> - create(StringRef Filename, LLVMContext &C); + create(const Twine &Filename, LLVMContext &C); /// \brief Create a sample profile reader from the supplied memory buffer. static ErrorOr<std::unique_ptr<SampleProfileReader>> create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C); + /// \brief Return the profile summary. + ProfileSummary &getSummary() { return *(Summary.get()); } + protected: /// \brief Map every function to its associated profile. /// @@ -283,6 +311,12 @@ protected: /// \brief Memory buffer holding the profile file. std::unique_ptr<MemoryBuffer> Buffer; + + /// \brief Profile summary information. + std::unique_ptr<ProfileSummary> Summary; + + /// \brief Compute summary for this profile. + void computeSummary(); }; class SampleProfileReaderText : public SampleProfileReader { @@ -348,6 +382,12 @@ protected: /// Function name table. std::vector<StringRef> NameTable; + +private: + std::error_code readSummaryEntry(std::vector<ProfileSummaryEntry> &Entries); + + /// \brief Read profile summary. + std::error_code readSummary(); }; typedef SmallVector<FunctionSamples *, 10> InlineCallStack; diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index 029dd2ebacb07..f6f2e2702e316 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -15,6 +15,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" @@ -31,10 +32,10 @@ class SampleProfileWriter { public: virtual ~SampleProfileWriter() {} - /// Write sample profiles in \p S for function \p FName. + /// Write sample profiles in \p S. /// /// \returns status code of the file update operation. - virtual std::error_code write(StringRef FName, const FunctionSamples &S) = 0; + virtual std::error_code write(const FunctionSamples &S) = 0; /// Write all the sample profiles in the given map of samples. /// @@ -42,11 +43,9 @@ public: std::error_code write(const StringMap<FunctionSamples> &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; - for (const auto &I : ProfileMap) { - StringRef FName = I.first(); const FunctionSamples &Profile = I.second; - if (std::error_code EC = write(FName, Profile)) + if (std::error_code EC = write(Profile)) return EC; } return sampleprof_error::success; @@ -75,12 +74,18 @@ protected: /// \brief Output stream where to emit the profile to. std::unique_ptr<raw_ostream> OutputStream; + + /// \brief Profile summary. + std::unique_ptr<ProfileSummary> Summary; + + /// \brief Compute summary for this profile. + void computeSummary(const StringMap<FunctionSamples> &ProfileMap); }; /// \brief Sample-based profile writer (text format). class SampleProfileWriterText : public SampleProfileWriter { public: - std::error_code write(StringRef FName, const FunctionSamples &S) override; + std::error_code write(const FunctionSamples &S) override; protected: SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) @@ -105,7 +110,7 @@ private: /// \brief Sample-based profile writer (binary format). class SampleProfileWriterBinary : public SampleProfileWriter { public: - std::error_code write(StringRef F, const FunctionSamples &S) override; + std::error_code write(const FunctionSamples &S) override; protected: SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) @@ -113,8 +118,9 @@ protected: std::error_code writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; + std::error_code writeSummary(); std::error_code writeNameIdx(StringRef FName); - std::error_code writeBody(StringRef FName, const FunctionSamples &S); + std::error_code writeBody(const FunctionSamples &S); private: void addName(StringRef FName); |