summaryrefslogtreecommitdiff
path: root/include/llvm/ProfileData
diff options
context:
space:
mode:
Diffstat (limited to 'include/llvm/ProfileData')
-rw-r--r--include/llvm/ProfileData/CoverageMapping.h113
-rw-r--r--include/llvm/ProfileData/InstrProf.h78
-rw-r--r--include/llvm/ProfileData/InstrProfData.inc67
-rw-r--r--include/llvm/ProfileData/SampleProf.h55
4 files changed, 190 insertions, 123 deletions
diff --git a/include/llvm/ProfileData/CoverageMapping.h b/include/llvm/ProfileData/CoverageMapping.h
index 3790e1358449..92a991eb39ba 100644
--- a/include/llvm/ProfileData/CoverageMapping.h
+++ b/include/llvm/ProfileData/CoverageMapping.h
@@ -20,13 +20,34 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/iterator.h"
+#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Endian.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/raw_ostream.h"
#include <system_error>
#include <tuple>
namespace llvm {
+namespace coverage {
+enum class coveragemap_error {
+ success = 0,
+ eof,
+ no_data_found,
+ unsupported_version,
+ truncated,
+ malformed
+};
+} // end of coverage namespace.
+}
+
+namespace std {
+template <>
+struct is_error_code_enum<llvm::coverage::coveragemap_error> : std::true_type {
+};
+}
+
+namespace llvm {
class IndexedInstrProfReader;
namespace coverage {
@@ -35,8 +56,6 @@ class CoverageMappingReader;
class CoverageMapping;
struct CounterExpressions;
-enum CoverageMappingVersion { CoverageMappingVersion1 };
-
/// \brief A Counter is an abstract value that describes how to compute the
/// execution count for a region of code using the collected profile count data.
struct Counter {
@@ -454,6 +473,76 @@ public:
CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion);
};
+const std::error_category &coveragemap_category();
+
+inline std::error_code make_error_code(coveragemap_error E) {
+ return std::error_code(static_cast<int>(E), coveragemap_category());
+}
+
+// Profile coverage map has the following layout:
+// [CoverageMapFileHeader]
+// [ArrayStart]
+// [CovMapFunctionRecord]
+// [CovMapFunctionRecord]
+// ...
+// [ArrayEnd]
+// [Encoded Region Mapping Data]
+LLVM_PACKED_START
+template <class IntPtrT> struct CovMapFunctionRecord {
+#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
+#include "llvm/ProfileData/InstrProfData.inc"
+
+ // Return the structural hash associated with the function.
+ template <support::endianness Endian> uint64_t getFuncHash() const {
+ return support::endian::byte_swap<uint64_t, Endian>(FuncHash);
+ }
+ // Return the coverage map data size for the funciton.
+ template <support::endianness Endian> uint32_t getDataSize() const {
+ return support::endian::byte_swap<uint32_t, Endian>(DataSize);
+ }
+ // Return function lookup key. The value is consider opaque.
+ template <support::endianness Endian> IntPtrT getFuncNameRef() const {
+ return support::endian::byte_swap<IntPtrT, Endian>(NamePtr);
+ }
+ // Return the PGO name of the function */
+ template <support::endianness Endian>
+ std::error_code getFuncName(InstrProfSymtab &ProfileNames,
+ StringRef &FuncName) const {
+ IntPtrT NameRef = getFuncNameRef<Endian>();
+ uint32_t NameS = support::endian::byte_swap<uint32_t, Endian>(NameSize);
+ FuncName = ProfileNames.getFuncName(NameRef, NameS);
+ if (NameS && FuncName.empty())
+ return coveragemap_error::malformed;
+ return std::error_code();
+ }
+};
+// Per module coverage mapping data header, i.e. CoverageMapFileHeader
+// documented above.
+struct CovMapHeader {
+#define COVMAP_HEADER(Type, LLVMType, Name, Init) Type Name;
+#include "llvm/ProfileData/InstrProfData.inc"
+ template <support::endianness Endian> uint32_t getNRecords() const {
+ return support::endian::byte_swap<uint32_t, Endian>(NRecords);
+ }
+ template <support::endianness Endian> uint32_t getFilenamesSize() const {
+ return support::endian::byte_swap<uint32_t, Endian>(FilenamesSize);
+ }
+ template <support::endianness Endian> uint32_t getCoverageSize() const {
+ return support::endian::byte_swap<uint32_t, Endian>(CoverageSize);
+ }
+ template <support::endianness Endian> uint32_t getVersion() const {
+ return support::endian::byte_swap<uint32_t, Endian>(Version);
+ }
+};
+
+LLVM_PACKED_END
+
+enum CoverageMappingVersion {
+ CoverageMappingVersion1 = 0,
+ // The current versin is Version1
+ CoverageMappingCurrentVersion = INSTR_PROF_COVMAP_VERSION
+};
+
} // end namespace coverage
/// \brief Provide DenseMapInfo for CounterExpression
@@ -484,26 +573,6 @@ template<> struct DenseMapInfo<coverage::CounterExpression> {
}
};
-const std::error_category &coveragemap_category();
-
-enum class coveragemap_error {
- success = 0,
- eof,
- no_data_found,
- unsupported_version,
- truncated,
- malformed
-};
-
-inline std::error_code make_error_code(coveragemap_error E) {
- return std::error_code(static_cast<int>(E), coveragemap_category());
-}
-
} // end namespace llvm
-namespace std {
-template <>
-struct is_error_code_enum<llvm::coveragemap_error> : std::true_type {};
-}
-
#endif // LLVM_PROFILEDATA_COVERAGEMAPPING_H_
diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h
index 49569d89507b..c84d8d24f206 100644
--- a/include/llvm/ProfileData/InstrProf.h
+++ b/include/llvm/ProfileData/InstrProf.h
@@ -30,7 +30,6 @@
#include <system_error>
#include <vector>
-#define INSTR_PROF_INDEX_VERSION 3
namespace llvm {
class Function;
@@ -66,7 +65,8 @@ inline StringRef getInstrProfValueProfFuncName() {
/// Return the name of the section containing function coverage mapping
/// data.
inline StringRef getInstrProfCoverageSectionName(bool AddSegment) {
- return AddSegment ? "__DATA,__llvm_covmap" : "__llvm_covmap";
+ return AddSegment ? "__DATA," INSTR_PROF_COVMAP_SECT_NAME_STR
+ : INSTR_PROF_COVMAP_SECT_NAME_STR;
}
/// Return the name prefix of variables containing instrumented function names.
@@ -89,6 +89,12 @@ inline StringRef getCoverageMappingVarName() {
return "__llvm_coverage_mapping";
}
+/// Return the name of the internal variable recording the array
+/// of PGO name vars referenced by the coverage mapping, The owning
+/// functions of those names are not emitted by FE (e.g, unused inline
+/// functions.)
+inline StringRef getCoverageNamesVarName() { return "__llvm_coverage_names"; }
+
/// Return the name of function that registers all the per-function control
/// data at program startup time by calling __llvm_register_function. This
/// function has internal linkage and is called by __llvm_profile_init
@@ -349,11 +355,14 @@ struct InstrProfValueSiteRecord {
return left.Value < right.Value;
});
}
+ /// Sort ValueData Descending by Count
+ inline void sortByCount();
/// Merge data from another InstrProfValueSiteRecord
/// Optionally scale merged counts by \p Weight.
- instrprof_error mergeValueData(InstrProfValueSiteRecord &Input,
- uint64_t Weight = 1);
+ instrprof_error merge(InstrProfValueSiteRecord &Input, uint64_t Weight = 1);
+ /// Scale up value profile data counts.
+ instrprof_error scale(uint64_t Weight);
};
/// Profiling information for a single function.
@@ -396,6 +405,19 @@ struct InstrProfRecord {
/// Optionally scale merged counts by \p Weight.
instrprof_error merge(InstrProfRecord &Other, uint64_t Weight = 1);
+ /// Scale up profile counts (including value profile data) by
+ /// \p Weight.
+ instrprof_error scale(uint64_t Weight);
+
+ /// Sort value profile data (per site) by count.
+ void sortValueData() {
+ for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
+ std::vector<InstrProfValueSiteRecord> &SiteRecords =
+ getValueSitesForKind(Kind);
+ for (auto &SR : SiteRecords)
+ SR.sortByCount();
+ }
+ }
/// Clear value data entries
void clearValueData() {
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
@@ -430,6 +452,8 @@ private:
// Scale merged value counts by \p Weight.
instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
uint64_t Weight);
+ // Scale up value profile data count.
+ instrprof_error scaleValueProfData(uint32_t ValueKind, uint64_t Weight);
};
uint32_t InstrProfRecord::getNumValueKinds() const {
@@ -497,11 +521,22 @@ inline support::endianness getHostEndianness() {
#define INSTR_PROF_VALUE_PROF_DATA
#include "llvm/ProfileData/InstrProfData.inc"
- /*
- * Initialize the record for runtime value profile data.
- * Return 0 if the initialization is successful, otherwise
- * return 1.
- */
+void InstrProfValueSiteRecord::sortByCount() {
+ ValueData.sort(
+ [](const InstrProfValueData &left, const InstrProfValueData &right) {
+ return left.Count > right.Count;
+ });
+ // Now truncate
+ size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
+ if (ValueData.size() > max_s)
+ ValueData.resize(max_s);
+}
+
+/*
+* Initialize the record for runtime value profile data.
+* Return 0 if the initialization is successful, otherwise
+* return 1.
+*/
int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
const uint16_t *NumValueSites,
ValueProfNode **Nodes);
@@ -597,31 +632,6 @@ struct Header {
} // end namespace RawInstrProf
-namespace coverage {
-
-// Profile coverage map has the following layout:
-// [CoverageMapFileHeader]
-// [ArrayStart]
-// [CovMapFunctionRecord]
-// [CovMapFunctionRecord]
-// ...
-// [ArrayEnd]
-// [Encoded Region Mapping Data]
-LLVM_PACKED_START
-template <class IntPtrT> struct CovMapFunctionRecord {
- #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
- #include "llvm/ProfileData/InstrProfData.inc"
-};
-// Per module coverage mapping data header, i.e. CoverageMapFileHeader
-// documented above.
-struct CovMapHeader {
-#define COVMAP_HEADER(Type, LLVMType, Name, Init) Type Name;
-#include "llvm/ProfileData/InstrProfData.inc"
-};
-
-LLVM_PACKED_END
-}
-
} // end namespace llvm
namespace std {
diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc
index 3a7c0c5f2773..33c7d94aea2a 100644
--- a/include/llvm/ProfileData/InstrProfData.inc
+++ b/include/llvm/ProfileData/InstrProfData.inc
@@ -28,7 +28,7 @@
*
* Examples of how the template is used to instantiate structure definition:
* 1. To declare a structure:
- *
+ *
* struct ProfData {
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* Type Name;
@@ -155,7 +155,7 @@ VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget)
#endif
COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
- llvm::Type::getInt8PtrTy(Ctx)))
+ llvm::Type::getInt8PtrTy(Ctx)))
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
NameValue.size()))
@@ -182,7 +182,7 @@ COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
COVMAP_HEADER(uint32_t, Int32Ty, Version, \
- llvm::ConstantInt::get(Int32Ty, CoverageMappingVersion1))
+ llvm::ConstantInt::get(Int32Ty, CoverageMappingCurrentVersion))
#undef COVMAP_HEADER
/* COVMAP_HEADER end. */
@@ -190,7 +190,8 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \
#ifdef INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_DATA_DEFINED
-/*!
+#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
+/*!
* This is the header of the data structure that defines the on-disk
* layout of the value profile data of a particular kind for one function.
*/
@@ -202,7 +203,7 @@ typedef struct ValueProfRecord {
* otherwise the record for this kind won't be emitted.
*/
uint32_t NumValueSites;
- /*
+ /*
* The first element of the array that stores the number of profiled
* values for each value site. The size of the array is NumValueSites.
* Since NumValueSites is greater than zero, there is at least one
@@ -226,7 +227,7 @@ typedef struct ValueProfRecord {
* \brief Return the number of value sites.
*/
uint32_t getNumValueSites() const { return NumValueSites; }
- /*!
+ /*!
* \brief Read data from this record and save it to Record.
*/
void deserializeTo(InstrProfRecord &Record,
@@ -247,10 +248,10 @@ typedef struct ValueProfRecord {
typedef struct ValueProfData {
/*
* Total size in bytes including this field. It must be a multiple
- * of sizeof(uint64_t).
+ * of sizeof(uint64_t).
*/
uint32_t TotalSize;
- /*
+ /*
*The number of value profile kinds that has value profile data.
* In this implementation, a value profile kind is considered to
* have profile data if the number of value profile sites for the
@@ -260,7 +261,7 @@ typedef struct ValueProfData {
*/
uint32_t NumValueKinds;
- /*
+ /*
* Following are a sequence of variable length records. The prefix/header
* of each record is defined by ValueProfRecord type. The number of
* records is NumValueKinds.
@@ -314,7 +315,7 @@ typedef struct ValueProfData {
#endif
} ValueProfData;
-/*
+/*
* The closure is designed to abstact away two types of value profile data:
* - InstrProfRecord which is the primary data structure used to
* represent profile data in host tools (reader, writer, and profile-use)
@@ -335,7 +336,7 @@ typedef struct ValueProfRecordClosure {
uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
- /*
+ /*
* After extracting the value profile data from the value profile record,
* this method is used to map the in-memory value to on-disk value. If
* the method is null, value will be written out untranslated.
@@ -346,7 +347,7 @@ typedef struct ValueProfRecordClosure {
ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
} ValueProfRecordClosure;
-/*
+/*
* A wrapper struct that represents value profile runtime data.
* Like InstrProfRecord class which is used by profiling host tools,
* ValueProfRuntimeRecord also implements the abstract intefaces defined in
@@ -384,7 +385,7 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
uint32_t getNumValueKindsRT(const void *R);
#undef INSTR_PROF_VALUE_PROF_DATA
-#endif /* INSTR_PROF_VALUE_PROF_DATA */
+#endif /* INSTR_PROF_VALUE_PROF_DATA */
#ifdef INSTR_PROF_COMMON_API_IMPL
@@ -412,7 +413,7 @@ uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
return Size;
}
-/*!
+/*!
* \brief Return the total size of the value profile record including the
* header and the value data.
*/
@@ -432,7 +433,7 @@ InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
This->NumValueSites));
}
-/*!
+/*!
* \brief Return the total number of value data for \c This record.
*/
INSTR_PROF_INLINE
@@ -444,7 +445,7 @@ uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
return NumValueData;
}
-/*!
+/*!
* \brief Use this method to advance to the next \c This \c ValueProfRecord.
*/
INSTR_PROF_INLINE
@@ -465,7 +466,7 @@ ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
/* Closure based interfaces. */
-/*!
+/*!
* Return the total size in bytes of the on-disk value profile data
* given the data stored in Record.
*/
@@ -535,7 +536,7 @@ ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
return VPD;
}
-/*
+/*
* The value profiler runtime library stores the value profile data
* for a given function in \c NumValueSites and \c Nodes structures.
* \c ValueProfRuntimeRecord class is used to encapsulate the runtime
@@ -639,7 +640,7 @@ static ValueProfRecordClosure RTRecordClosure = {0,
getValueForSiteRT,
allocValueProfDataRT};
-/*
+/*
* Return the size of ValueProfData structure to store data
* recorded in the runtime record.
*/
@@ -648,7 +649,7 @@ uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) {
return getValueProfDataSize(&RTRecordClosure);
}
-/*
+/*
* Return a ValueProfData instance that stores the data collected
* from runtime. If \c DstData is provided by the caller, the value
* profile data will be store in *DstData and DstData is returned,
@@ -696,18 +697,31 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
/* Raw profile format version. */
#define INSTR_PROF_RAW_VERSION 2
+#define INSTR_PROF_INDEX_VERSION 3
+#define INSTR_PROF_COVMAP_VERSION 0
+
+/* Profile version is always of type uint_64_t. Reserve the upper 8 bits in the
+ * version for other variants of profile. We set the lowest bit of the upper 8
+ * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
+ * generated profile, and 0 if this is a Clang FE generated profile.
+*/
+#define VARIANT_MASKS_ALL 0xff00000000000000ULL
+#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
/* Runtime section names and name strings. */
#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data
#define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names
#define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts
+#define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap
-#define INSTR_PROF_DATA_SECT_NAME_STR \
- INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME)
-#define INSTR_PROF_NAME_SECT_NAME_STR \
- INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME)
-#define INSTR_PROF_CNTS_SECT_NAME_STR \
- INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
+#define INSTR_PROF_DATA_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME)
+#define INSTR_PROF_NAME_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME)
+#define INSTR_PROF_CNTS_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
+#define INSTR_PROF_COVMAP_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME)
/* Macros to define start/stop section symbol for a given
* section on Linux. For instance
@@ -751,4 +765,3 @@ typedef struct ValueProfNode {
#else
#undef INSTR_PROF_DATA_DEFINED
#endif
-
diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h
index 8df3fe803209..6c39cf9458dc 100644
--- a/include/llvm/ProfileData/SampleProf.h
+++ b/include/llvm/ProfileData/SampleProf.h
@@ -140,16 +140,9 @@ public:
/// around unsigned integers.
sampleprof_error addSamples(uint64_t S, uint64_t Weight = 1) {
bool Overflowed;
- if (Weight > 1) {
- S = SaturatingMultiply(S, Weight, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
- }
- NumSamples = SaturatingAdd(NumSamples, S, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
-
- return sampleprof_error::success;
+ NumSamples = SaturatingMultiplyAdd(S, Weight, NumSamples, &Overflowed);
+ return Overflowed ? sampleprof_error::counter_overflow
+ : sampleprof_error::success;
}
/// Add called function \p F with samples \p S.
@@ -161,16 +154,10 @@ public:
uint64_t Weight = 1) {
uint64_t &TargetSamples = CallTargets[F];
bool Overflowed;
- if (Weight > 1) {
- S = SaturatingMultiply(S, Weight, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
- }
- TargetSamples = SaturatingAdd(TargetSamples, S, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
-
- return sampleprof_error::success;
+ TargetSamples =
+ SaturatingMultiplyAdd(S, Weight, TargetSamples, &Overflowed);
+ return Overflowed ? sampleprof_error::counter_overflow
+ : sampleprof_error::success;
}
/// Return true if this sample record contains function calls.
@@ -215,29 +202,17 @@ public:
void dump() const;
sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) {
bool Overflowed;
- if (Weight > 1) {
- Num = SaturatingMultiply(Num, Weight, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
- }
- TotalSamples = SaturatingAdd(TotalSamples, Num, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
-
- return sampleprof_error::success;
+ TotalSamples =
+ SaturatingMultiplyAdd(Num, Weight, TotalSamples, &Overflowed);
+ return Overflowed ? sampleprof_error::counter_overflow
+ : sampleprof_error::success;
}
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
bool Overflowed;
- if (Weight > 1) {
- Num = SaturatingMultiply(Num, Weight, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
- }
- TotalHeadSamples = SaturatingAdd(TotalHeadSamples, Num, &Overflowed);
- if (Overflowed)
- return sampleprof_error::counter_overflow;
-
- return sampleprof_error::success;
+ TotalHeadSamples =
+ SaturatingMultiplyAdd(Num, Weight, TotalHeadSamples, &Overflowed);
+ return Overflowed ? sampleprof_error::counter_overflow
+ : sampleprof_error::success;
}
sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator,
uint64_t Num, uint64_t Weight = 1) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-04 15:18:13 +0000