diff options
Diffstat (limited to 'include/llvm/ProfileData')
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMapping.h | 7 | ||||
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMappingReader.h | 17 | ||||
| -rw-r--r-- | include/llvm/ProfileData/Coverage/CoverageMappingWriter.h | 7 | ||||
| -rw-r--r-- | include/llvm/ProfileData/GCOV.h | 19 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProf.h | 109 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfData.inc | 92 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfReader.h | 56 | ||||
| -rw-r--r-- | include/llvm/ProfileData/InstrProfWriter.h | 38 | ||||
| -rw-r--r-- | include/llvm/ProfileData/ProfileCommon.h | 10 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProf.h | 50 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProfReader.h | 16 | ||||
| -rw-r--r-- | include/llvm/ProfileData/SampleProfWriter.h | 7 |
12 files changed, 308 insertions, 120 deletions
diff --git a/include/llvm/ProfileData/Coverage/CoverageMapping.h b/include/llvm/ProfileData/Coverage/CoverageMapping.h index beaa36553287..11758ac4cf2f 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -1,9 +1,8 @@ //===- CoverageMapping.h - Code coverage mapping support --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index c88c71a6d6f4..57a2aaefd660 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -1,9 +1,8 @@ //===- CoverageMappingReader.h - Code coverage mapping reader ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -204,9 +203,15 @@ public: BinaryCoverageReader(const BinaryCoverageReader &) = delete; BinaryCoverageReader &operator=(const BinaryCoverageReader &) = delete; + static Expected<std::vector<std::unique_ptr<BinaryCoverageReader>>> + create(MemoryBufferRef ObjectBuffer, StringRef Arch, + SmallVectorImpl<std::unique_ptr<MemoryBuffer>> &ObjectFileBuffers); + static Expected<std::unique_ptr<BinaryCoverageReader>> - create(std::unique_ptr<MemoryBuffer> &ObjectBuffer, - StringRef Arch); + createCoverageReaderFromBuffer(StringRef Coverage, + InstrProfSymtab &&ProfileNames, + uint8_t BytesInAddress, + support::endianness Endian); Error readNextRecord(CoverageMappingRecord &Record) override; }; diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h index 86fb1bdf1773..5f88cacdfcbb 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h @@ -1,9 +1,8 @@ //===- CoverageMappingWriter.h - Code coverage mapping writer ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ProfileData/GCOV.h b/include/llvm/ProfileData/GCOV.h index a088f63a6915..004ff3f4a2e2 100644 --- a/include/llvm/ProfileData/GCOV.h +++ b/include/llvm/ProfileData/GCOV.h @@ -1,9 +1,8 @@ //===- GCOV.h - LLVM coverage tool ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,9 +44,10 @@ enum GCOVVersion { V402, V404, V704 }; /// A struct for passing gcov options between functions. struct Options { - Options(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N) + Options(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N, bool X) : AllBlocks(A), BranchInfo(B), BranchCount(C), FuncCoverage(F), - PreservePaths(P), UncondBranch(U), LongFileNames(L), NoOutput(N) {} + PreservePaths(P), UncondBranch(U), LongFileNames(L), NoOutput(N), + HashFilenames(X) {} bool AllBlocks; bool BranchInfo; @@ -57,6 +57,7 @@ struct Options { bool UncondBranch; bool LongFileNames; bool NoOutput; + bool HashFilenames; }; } // end namespace GCOV @@ -317,12 +318,6 @@ class GCOVBlock { uint64_t Count = 0; }; - struct SortDstEdgesFunctor { - bool operator()(const GCOVEdge *E1, const GCOVEdge *E2) { - return E1->Dst.Number < E2->Dst.Number; - } - }; - public: using EdgeIterator = SmallVectorImpl<GCOVEdge *>::const_iterator; using BlockVector = SmallVector<const GCOVBlock *, 4>; diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index dc45021fc47d..c7d764ade30d 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -1,9 +1,8 @@ //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -235,7 +234,7 @@ bool isIRPGOFlagSet(const Module *M); bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false); enum InstrProfValueKind : uint32_t { -#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, #include "llvm/ProfileData/InstrProfData.inc" }; @@ -591,6 +590,70 @@ StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { return PGOName.drop_front(S + 1); } +// To store the sums of profile count values, or the percentage of +// the sums of the total count values. +struct CountSumOrPercent { + uint64_t NumEntries; + double CountSum; + double ValueCounts[IPVK_Last - IPVK_First + 1]; + CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {} + void reset() { + NumEntries = 0; + CountSum = 0.0f; + for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) + ValueCounts[I] = 0.0f; + } +}; + +// Function level or program level overlap information. +struct OverlapStats { + enum OverlapStatsLevel { ProgramLevel, FunctionLevel }; + // Sum of the total count values for the base profile. + CountSumOrPercent Base; + // Sum of the total count values for the test profile. + CountSumOrPercent Test; + // Overlap lap score. Should be in range of [0.0f to 1.0f]. + CountSumOrPercent Overlap; + CountSumOrPercent Mismatch; + CountSumOrPercent Unique; + OverlapStatsLevel Level; + const std::string *BaseFilename; + const std::string *TestFilename; + StringRef FuncName; + uint64_t FuncHash; + bool Valid; + + OverlapStats(OverlapStatsLevel L = ProgramLevel) + : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0), + Valid(false) {} + + void dump(raw_fd_ostream &OS) const; + + void setFuncInfo(StringRef Name, uint64_t Hash) { + FuncName = Name; + FuncHash = Hash; + } + + Error accumuateCounts(const std::string &BaseFilename, + const std::string &TestFilename, bool IsCS); + void addOneMismatch(const CountSumOrPercent &MismatchFunc); + void addOneUnique(const CountSumOrPercent &UniqueFunc); + + static inline double score(uint64_t Val1, uint64_t Val2, double Sum1, + double Sum2) { + if (Sum1 < 1.0f || Sum2 < 1.0f) + return 0.0f; + return std::min(Val1 / Sum1, Val2 / Sum2); + } +}; + +// This is used to filter the functions whose overlap information +// to be output. +struct OverlapFuncFilters { + uint64_t ValueCutoff; + const std::string NameFilter; +}; + struct InstrProfValueSiteRecord { /// Value profiling data pairs at a given value site. std::list<InstrProfValueData> ValueData; @@ -616,6 +679,10 @@ struct InstrProfValueSiteRecord { function_ref<void(instrprof_error)> Warn); /// Scale up value profile data counts. void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn); + + /// Compute the overlap b/w this record and Input record. + void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, + OverlapStats &Overlap, OverlapStats &FuncLevelOverlap); }; /// Profiling information for a single function. @@ -704,6 +771,18 @@ struct InstrProfRecord { /// Clear value data entries void clearValueData() { ValueData = nullptr; } + /// Compute the sums of all counts and store in Sum. + void accumuateCounts(CountSumOrPercent &Sum) const; + + /// Compute the overlap b/w this IntrprofRecord and Other. + void overlap(InstrProfRecord &Other, OverlapStats &Overlap, + OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff); + + /// Compute the overlap of value profile counts. + void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, + OverlapStats &Overlap, + OverlapStats &FuncLevelOverlap); + private: struct ValueProfData { std::vector<InstrProfValueSiteRecord> IndirectCallSites; @@ -768,10 +847,20 @@ struct NamedInstrProfRecord : InstrProfRecord { StringRef Name; uint64_t Hash; + // We reserve this bit as the flag for context sensitive profile record. + static const int CS_FLAG_IN_FUNC_HASH = 60; + NamedInstrProfRecord() = default; NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector<uint64_t> Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} + + static bool hasCSFlagInHash(uint64_t FuncHash) { + return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); + } + static void setCSFlagInHash(uint64_t &FuncHash) { + FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH); + } }; uint32_t InstrProfRecord::getNumValueKinds() const { @@ -1005,6 +1094,8 @@ namespace RawInstrProf { // from control data struct is changed from raw pointer to Name's MD5 value. // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the // raw header. +// Version 5: Bit 60 of FuncHash is reserved for the flag for the context +// sensitive records. const uint64_t Version = INSTR_PROF_RAW_VERSION; template <class IntPtrT> inline uint64_t getMagic(); @@ -1041,6 +1132,12 @@ struct Header { void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast); -} // end namespace llvm +// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime +// aware this is an ir_level profile so it can set the version flag. +void createIRLevelProfileFlagVar(Module &M, bool IsCS); + +// Create the variable for the profile file name. +void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); +} // end namespace llvm #endif // LLVM_PROFILEDATA_INSTRPROF_H diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc index 454620ed997a..749781b9ac2d 100644 --- a/include/llvm/ProfileData/InstrProfData.inc +++ b/include/llvm/ProfileData/InstrProfData.inc @@ -1,9 +1,8 @@ /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -170,7 +169,7 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) /* VALUE_PROF_KIND start */ #ifndef VALUE_PROF_KIND -#define VALUE_PROF_KIND(Enumerator, Value) +#define VALUE_PROF_KIND(Enumerator, Value, Descr) #else #define INSTR_PROF_DATA_DEFINED #endif @@ -183,16 +182,16 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) * For this remapping the ProfData is used. ProfData contains both the function * name hash and the function address. */ -VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ -VALUE_PROF_KIND(IPVK_MemOPSize, 1) +VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ -VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -250,22 +249,25 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_COFF), "__DATA,") + INSTR_PROF_DATA_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COFF), "__DATA,") + INSTR_PROF_CNTS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_COFF), "__DATA,") + INSTR_PROF_NAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_COFF), "__DATA,") + INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COFF), "__DATA,") + INSTR_PROF_VNODES_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COFF), "__LLVM_COV,") + INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -636,10 +638,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 57 indicates there are context-sensitive records in the profile. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime @@ -655,13 +659,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap -/* Win32 */ -#define INSTR_PROF_DATA_COFF .lprfd -#define INSTR_PROF_NAME_COFF .lprfn -#define INSTR_PROF_CNTS_COFF .lprfc -#define INSTR_PROF_VALS_COFF .lprfv -#define INSTR_PROF_VNODES_COFF .lprfnd -#define INSTR_PROF_COVMAP_COFF .lcovmap +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +/* Windows section names. Because these section names contain dollar characters, + * they must be quoted. + */ +#define INSTR_PROF_DATA_COFF ".lprfd$M" +#define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_VALS_COFF ".lprfv$M" +#define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -675,32 +683,30 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COMMON -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COMMON -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COMMON +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COMMON +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) /* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COMMON -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COMMON +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +/* Order file instrumentation. */ +#define INSTR_PROF_ORDERFILE_SECT_NAME \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) #endif -#define INSTR_PROF_DATA_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) -#define INSTR_PROF_NAME_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) -#define INSTR_PROF_CNTS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) -#define INSTR_PROF_COVMAP_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) -#define INSTR_PROF_VALS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME) -#define INSTR_PROF_VNODES_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer +#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -735,6 +741,12 @@ typedef struct InstrProfValueData { #endif /* INSTR_PROF_DATA_INC */ +#ifndef INSTR_ORDER_FILE_INC +// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). +#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 +#define INSTR_ORDER_FILE_BUFFER_BITS 17 +#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff +#endif /* INSTR_ORDER_FILE_INC */ #else #undef INSTR_PROF_DATA_DEFINED #endif diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h index 08d782276117..73751faab88e 100644 --- a/include/llvm/ProfileData/InstrProfReader.h +++ b/include/llvm/ProfileData/InstrProfReader.h @@ -1,9 +1,8 @@ //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -78,6 +77,8 @@ public: virtual bool isIRLevelProfile() const = 0; + virtual bool hasCSIRLevelProfile() const = 0; + /// Return the PGO symtab. There are three different readers: /// Raw, Text, and Indexed profile readers. The first two types /// of readers are used only by llvm-profdata tool, while the indexed @@ -90,6 +91,9 @@ public: /// compiler. virtual InstrProfSymtab &getSymtab() = 0; + /// Compute the sum of counts and return in Sum. + void accumuateCounts(CountSumOrPercent &Sum, bool IsCS); + protected: std::unique_ptr<InstrProfSymtab> Symtab; @@ -143,6 +147,7 @@ private: /// Iterator over the profile data. line_iterator Line; bool IsIRLevelProfile = false; + bool HasCSIRLevelProfile = false; Error readValueProfileData(InstrProfRecord &Record); @@ -157,6 +162,8 @@ public: bool isIRLevelProfile() const override { return IsIRLevelProfile; } + bool hasCSIRLevelProfile() const override { return HasCSIRLevelProfile; } + /// Read the header. Error readHeader() override; @@ -213,6 +220,10 @@ public: return (Version & VARIANT_MASK_IR_PROF) != 0; } + bool hasCSIRLevelProfile() const override { + return (Version & VARIANT_MASK_CSIR_PROF) != 0; + } + InstrProfSymtab &getSymtab() override { assert(Symtab.get()); return *Symtab.get(); @@ -342,6 +353,7 @@ struct InstrProfReaderIndexBase { virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; virtual uint64_t getVersion() const = 0; virtual bool isIRLevelProfile() const = 0; + virtual bool hasCSIRLevelProfile() const = 0; virtual Error populateSymtab(InstrProfSymtab &) = 0; }; @@ -386,6 +398,10 @@ public: return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; } + bool hasCSIRLevelProfile() const override { + return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0; + } + Error populateSymtab(InstrProfSymtab &Symtab) override { return Symtab.create(HashTable->keys()); } @@ -413,13 +429,16 @@ private: std::unique_ptr<InstrProfReaderRemapper> Remapper; /// Profile summary data. std::unique_ptr<ProfileSummary> Summary; + /// Context sensitive profile summary data. + std::unique_ptr<ProfileSummary> CS_Summary; // Index to the current record in the record array. unsigned RecordIndex; // Read the profile summary. Return a pointer pointing to one byte past the // end of the summary data if it exists or the input \c Cur. + // \c UseCS indicates whether to use the context-sensitive profile summary. const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version, - const unsigned char *Cur); + const unsigned char *Cur, bool UseCS); public: IndexedInstrProfReader( @@ -433,6 +452,9 @@ public: /// Return the profile version. uint64_t getVersion() const { return Index->getVersion(); } bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } + bool hasCSIRLevelProfile() const override { + return Index->hasCSIRLevelProfile(); + } /// Return true if the given buffer is in an indexed instrprof format. static bool hasFormat(const MemoryBuffer &DataBuffer); @@ -451,7 +473,16 @@ public: std::vector<uint64_t> &Counts); /// Return the maximum of all known function counts. - uint64_t getMaximumFunctionCount() { return Summary->getMaxFunctionCount(); } + /// \c UseCS indicates whether to use the context-sensitive count. + uint64_t getMaximumFunctionCount(bool UseCS) { + if (UseCS) { + assert(CS_Summary && "No context sensitive profile summary"); + return CS_Summary->getMaxFunctionCount(); + } else { + assert(Summary && "No profile summary"); + return Summary->getMaxFunctionCount(); + } + } /// Factory method to create an indexed reader. static Expected<std::unique_ptr<IndexedInstrProfReader>> @@ -470,7 +501,18 @@ public: // to be used by llvm-profdata (for dumping). Avoid using this when // the client is the compiler. InstrProfSymtab &getSymtab() override; - ProfileSummary &getSummary() { return *(Summary.get()); } + + /// Return the profile summary. + /// \c UseCS indicates whether to use the context-sensitive summary. + ProfileSummary &getSummary(bool UseCS) { + if (UseCS) { + assert(CS_Summary && "No context sensitive summary"); + return *(CS_Summary.get()); + } else { + assert(Summary && "No profile summary"); + return *(Summary.get()); + } + } }; } // end namespace llvm diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h index 8107ab386fe2..5882fa2781e2 100644 --- a/include/llvm/ProfileData/InstrProfWriter.h +++ b/include/llvm/ProfileData/InstrProfWriter.h @@ -1,9 +1,8 @@ //===- InstrProfWriter.h - Instrumented profiling writer --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,7 +33,8 @@ class raw_fd_ostream; class InstrProfWriter { public: using ProfilingData = SmallDenseMap<uint64_t, InstrProfRecord>; - enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; + // PF_IRLevelWithCS is the profile from context sensitive IR instrumentation. + enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel, PF_IRLevelWithCS }; private: bool Sparse; @@ -75,20 +75,36 @@ public: std::unique_ptr<MemoryBuffer> writeBuffer(); /// Set the ProfileKind. Report error if mixing FE and IR level profiles. - Error setIsIRLevelProfile(bool IsIRLevel) { + /// \c WithCS indicates if this is for contenxt sensitive instrumentation. + Error setIsIRLevelProfile(bool IsIRLevel, bool WithCS) { if (ProfileKind == PF_Unknown) { - ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE; + if (IsIRLevel) + ProfileKind = WithCS ? PF_IRLevelWithCS : PF_IRLevel; + else + ProfileKind = PF_FE; return Error::success(); } - return (IsIRLevel == (ProfileKind == PF_IRLevel)) - ? Error::success() - : make_error<InstrProfError>( - instrprof_error::unsupported_version); + + if (((ProfileKind != PF_FE) && !IsIRLevel) || + ((ProfileKind == PF_FE) && IsIRLevel)) + return make_error<InstrProfError>(instrprof_error::unsupported_version); + + // When merging a context-sensitive profile (WithCS == true) with an IRLevel + // profile, set the kind to PF_IRLevelWithCS. + if (ProfileKind == PF_IRLevel && WithCS) + ProfileKind = PF_IRLevelWithCS; + + return Error::success(); } // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); void setOutputSparse(bool Sparse); + // Compute the overlap b/w this object and Other. Program level result is + // stored in Overlap and function level result is stored in FuncLevelOverlap. + void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, + OverlapStats &FuncLevelOverlap, + const OverlapFuncFilters &FuncFilter); private: void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I, diff --git a/include/llvm/ProfileData/ProfileCommon.h b/include/llvm/ProfileData/ProfileCommon.h index 087588f06340..f98a34387fdf 100644 --- a/include/llvm/ProfileData/ProfileCommon.h +++ b/include/llvm/ProfileData/ProfileCommon.h @@ -1,9 +1,8 @@ //===- ProfileCommon.h - Common profiling APIs. -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -84,7 +83,8 @@ public: SampleProfileSummaryBuilder(std::vector<uint32_t> Cutoffs) : ProfileSummaryBuilder(std::move(Cutoffs)) {} - void addRecord(const sampleprof::FunctionSamples &FS); + void addRecord(const sampleprof::FunctionSamples &FS, + bool isCallsiteSample = false); std::unique_ptr<ProfileSummary> getSummary(); }; diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 927dfd246878..7fbc857b7230 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -1,9 +1,8 @@ //===- SampleProf.h - Sampling profiling format support ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -411,6 +410,34 @@ public: return getNameInModule(Name, M); } + /// Return the canonical name for a function, taking into account + /// suffix elision policy attributes. + static StringRef getCanonicalFnName(const Function &F) { + static const char *knownSuffixes[] = { ".llvm.", ".part." }; + auto AttrName = "sample-profile-suffix-elision-policy"; + auto Attr = F.getFnAttribute(AttrName).getValueAsString(); + if (Attr == "" || Attr == "all") { + return F.getName().split('.').first; + } else if (Attr == "selected") { + StringRef Cand(F.getName()); + for (const auto &Suf : knownSuffixes) { + StringRef Suffix(Suf); + auto It = Cand.rfind(Suffix); + if (It == StringRef::npos) + return Cand; + auto Dit = Cand.rfind('.'); + if (Dit == It + Suffix.size() - 1) + Cand = Cand.substr(0, It); + } + return Cand; + } else if (Attr == "none") { + return F.getName(); + } else { + assert(false && "internal error: unknown suffix elision policy"); + } + return F.getName(); + } + /// Translate \p Name into its original name in Module. /// When the Format is not SPF_Compact_Binary, \p Name needs no translation. /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples @@ -466,11 +493,9 @@ public: /// built in post-thin-link phase and var promotion has been done, /// we need to add the substring of function name without the suffix /// into the GUIDToFuncNameMap. - auto pos = OrigName.find('.'); - if (pos != StringRef::npos) { - StringRef NewName = OrigName.substr(0, pos); - GUIDToFuncNameMap.insert({Function::getGUID(NewName), NewName}); - } + StringRef CanonName = getCanonicalFnName(F); + if (CanonName != OrigName) + GUIDToFuncNameMap.insert({Function::getGUID(CanonName), CanonName}); } CurrentModule = &M; } @@ -547,10 +572,9 @@ public: SampleSorter(const std::map<LocationT, SampleT> &Samples) { for (const auto &I : Samples) V.push_back(&I); - std::stable_sort(V.begin(), V.end(), - [](const SamplesWithLoc *A, const SamplesWithLoc *B) { - return A->first < B->first; - }); + llvm::stable_sort(V, [](const SamplesWithLoc *A, const SamplesWithLoc *B) { + return A->first < B->first; + }); } const SamplesWithLocList &get() const { return V; } diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index 5cc729e42cc8..969cdea859c9 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -1,9 +1,8 @@ //===- SampleProfReader.h - Read LLVM sample profile data -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -287,10 +286,11 @@ public: /// Return the samples collected for function \p F. FunctionSamples *getSamplesFor(const Function &F) { - // The function name may have been updated by adding suffix. In sample - // profile, the function names are all stripped, so we need to strip - // the function name suffix before matching with profile. - return getSamplesFor(F.getName().split('.').first); + // The function name may have been updated by adding suffix. Call + // a helper to (optionally) strip off suffixes so that we can + // match against the original function name in the profile. + StringRef CanonName = FunctionSamples::getCanonicalFnName(F); + return getSamplesFor(CanonName); } /// Return the samples collected for function \p F. diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index d5ac6e53e4f7..81e6e3ab0b4a 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -1,9 +1,8 @@ //===- SampleProfWriter.h - Write LLVM sample profile data ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // |