aboutsummaryrefslogtreecommitdiff
path: root/lib/ProfileData
diff options
context:
space:
mode:
Diffstat (limited to 'lib/ProfileData')
-rw-r--r--lib/ProfileData/CMakeLists.txt6
-rw-r--r--lib/ProfileData/Coverage/CMakeLists.txt11
-rw-r--r--lib/ProfileData/Coverage/CoverageMapping.cpp (renamed from lib/ProfileData/CoverageMapping.cpp)260
-rw-r--r--lib/ProfileData/Coverage/CoverageMappingReader.cpp (renamed from lib/ProfileData/CoverageMappingReader.cpp)432
-rw-r--r--lib/ProfileData/Coverage/CoverageMappingWriter.cpp (renamed from lib/ProfileData/CoverageMappingWriter.cpp)2
-rw-r--r--lib/ProfileData/Coverage/LLVMBuild.txt23
-rw-r--r--lib/ProfileData/InstrProf.cpp469
-rw-r--r--lib/ProfileData/InstrProfReader.cpp289
-rw-r--r--lib/ProfileData/InstrProfWriter.cpp221
-rw-r--r--lib/ProfileData/LLVMBuild.txt5
-rw-r--r--lib/ProfileData/Makefile14
-rw-r--r--lib/ProfileData/ProfileSummaryBuilder.cpp116
-rw-r--r--lib/ProfileData/SampleProf.cpp24
-rw-r--r--lib/ProfileData/SampleProfReader.cpp97
-rw-r--r--lib/ProfileData/SampleProfWriter.cpp60
15 files changed, 1414 insertions, 615 deletions
diff --git a/lib/ProfileData/CMakeLists.txt b/lib/ProfileData/CMakeLists.txt
index 22cca4b44df5..cd65762ae6a0 100644
--- a/lib/ProfileData/CMakeLists.txt
+++ b/lib/ProfileData/CMakeLists.txt
@@ -2,9 +2,7 @@ add_llvm_library(LLVMProfileData
InstrProf.cpp
InstrProfReader.cpp
InstrProfWriter.cpp
- CoverageMapping.cpp
- CoverageMappingWriter.cpp
- CoverageMappingReader.cpp
+ ProfileSummaryBuilder.cpp
SampleProf.cpp
SampleProfReader.cpp
SampleProfWriter.cpp
@@ -15,3 +13,5 @@ add_llvm_library(LLVMProfileData
DEPENDS
intrinsics_gen
)
+
+add_subdirectory(Coverage)
diff --git a/lib/ProfileData/Coverage/CMakeLists.txt b/lib/ProfileData/Coverage/CMakeLists.txt
new file mode 100644
index 000000000000..035b8fdb8b3d
--- /dev/null
+++ b/lib/ProfileData/Coverage/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_llvm_library(LLVMCoverage
+ CoverageMapping.cpp
+ CoverageMappingWriter.cpp
+ CoverageMappingReader.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${LLVM_MAIN_INCLUDE_DIR}/llvm/ProfileData/Coverage
+
+ DEPENDS
+ intrinsics_gen
+ )
diff --git a/lib/ProfileData/CoverageMapping.cpp b/lib/ProfileData/Coverage/CoverageMapping.cpp
index f5d477bd139a..fcd4e24bdfcb 100644
--- a/lib/ProfileData/CoverageMapping.cpp
+++ b/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -12,11 +12,11 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/ProfileData/CoverageMapping.h"
+#include "llvm/ProfileData/Coverage/CoverageMapping.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallBitVector.h"
-#include "llvm/ProfileData/CoverageMappingReader.h"
+#include "llvm/ProfileData/Coverage/CoverageMappingReader.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Errc.h"
@@ -143,28 +143,30 @@ void CounterMappingContext::dump(const Counter &C,
}
if (CounterValues.empty())
return;
- ErrorOr<int64_t> Value = evaluate(C);
- if (!Value)
+ Expected<int64_t> Value = evaluate(C);
+ if (auto E = Value.takeError()) {
+ llvm::consumeError(std::move(E));
return;
+ }
OS << '[' << *Value << ']';
}
-ErrorOr<int64_t> CounterMappingContext::evaluate(const Counter &C) const {
+Expected<int64_t> CounterMappingContext::evaluate(const Counter &C) const {
switch (C.getKind()) {
case Counter::Zero:
return 0;
case Counter::CounterValueReference:
if (C.getCounterID() >= CounterValues.size())
- return make_error_code(errc::argument_out_of_domain);
+ return errorCodeToError(errc::argument_out_of_domain);
return CounterValues[C.getCounterID()];
case Counter::Expression: {
if (C.getExpressionID() >= Expressions.size())
- return make_error_code(errc::argument_out_of_domain);
+ return errorCodeToError(errc::argument_out_of_domain);
const auto &E = Expressions[C.getExpressionID()];
- ErrorOr<int64_t> LHS = evaluate(E.LHS);
+ Expected<int64_t> LHS = evaluate(E.LHS);
if (!LHS)
return LHS;
- ErrorOr<int64_t> RHS = evaluate(E.RHS);
+ Expected<int64_t> RHS = evaluate(E.RHS);
if (!RHS)
return RHS;
return E.Kind == CounterExpression::Subtract ? *LHS - *RHS : *LHS + *RHS;
@@ -181,7 +183,7 @@ void FunctionRecordIterator::skipOtherFiles() {
*this = FunctionRecordIterator();
}
-ErrorOr<std::unique_ptr<CoverageMapping>>
+Expected<std::unique_ptr<CoverageMapping>>
CoverageMapping::load(CoverageMappingReader &CoverageReader,
IndexedInstrProfReader &ProfileReader) {
auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping());
@@ -191,13 +193,14 @@ CoverageMapping::load(CoverageMappingReader &CoverageReader,
CounterMappingContext Ctx(Record.Expressions);
Counts.clear();
- if (std::error_code EC = ProfileReader.getFunctionCounts(
+ if (Error E = ProfileReader.getFunctionCounts(
Record.FunctionName, Record.FunctionHash, Counts)) {
- if (EC == instrprof_error::hash_mismatch) {
+ instrprof_error IPE = InstrProfError::take(std::move(E));
+ if (IPE == instrprof_error::hash_mismatch) {
Coverage->MismatchedFunctionCount++;
continue;
- } else if (EC != instrprof_error::unknown_function)
- return EC;
+ } else if (IPE != instrprof_error::unknown_function)
+ return make_error<InstrProfError>(IPE);
Counts.assign(Record.MappingRegions.size(), 0);
}
Ctx.setCounts(Counts);
@@ -205,14 +208,18 @@ CoverageMapping::load(CoverageMappingReader &CoverageReader,
assert(!Record.MappingRegions.empty() && "Function has no regions");
StringRef OrigFuncName = Record.FunctionName;
- if (!Record.Filenames.empty())
+ if (Record.Filenames.empty())
+ OrigFuncName = getFuncNameWithoutPrefix(OrigFuncName);
+ else
OrigFuncName =
getFuncNameWithoutPrefix(OrigFuncName, Record.Filenames[0]);
FunctionRecord Function(OrigFuncName, Record.Filenames);
for (const auto &Region : Record.MappingRegions) {
- ErrorOr<int64_t> ExecutionCount = Ctx.evaluate(Region.Count);
- if (!ExecutionCount)
+ Expected<int64_t> ExecutionCount = Ctx.evaluate(Region.Count);
+ if (auto E = ExecutionCount.takeError()) {
+ llvm::consumeError(std::move(E));
break;
+ }
Function.pushRegion(Region, *ExecutionCount);
}
if (Function.CountedRegions.size() != Record.MappingRegions.size()) {
@@ -226,20 +233,20 @@ CoverageMapping::load(CoverageMappingReader &CoverageReader,
return std::move(Coverage);
}
-ErrorOr<std::unique_ptr<CoverageMapping>>
+Expected<std::unique_ptr<CoverageMapping>>
CoverageMapping::load(StringRef ObjectFilename, StringRef ProfileFilename,
StringRef Arch) {
auto CounterMappingBuff = MemoryBuffer::getFileOrSTDIN(ObjectFilename);
if (std::error_code EC = CounterMappingBuff.getError())
- return EC;
+ return errorCodeToError(EC);
auto CoverageReaderOrErr =
BinaryCoverageReader::create(CounterMappingBuff.get(), Arch);
- if (std::error_code EC = CoverageReaderOrErr.getError())
- return EC;
+ if (Error E = CoverageReaderOrErr.takeError())
+ return std::move(E);
auto CoverageReader = std::move(CoverageReaderOrErr.get());
auto ProfileReaderOrErr = IndexedInstrProfReader::create(ProfileFilename);
- if (auto EC = ProfileReaderOrErr.getError())
- return EC;
+ if (Error E = ProfileReaderOrErr.takeError())
+ return std::move(E);
auto ProfileReader = std::move(ProfileReaderOrErr.get());
return load(*CoverageReader, *ProfileReader);
}
@@ -270,9 +277,11 @@ public:
};
class SegmentBuilder {
- std::vector<CoverageSegment> Segments;
+ std::vector<CoverageSegment> &Segments;
SmallVector<const CountedRegion *, 8> ActiveRegions;
+ SegmentBuilder(std::vector<CoverageSegment> &Segments) : Segments(Segments) {}
+
/// Start a segment with no count specified.
void startSegment(unsigned Line, unsigned Col) {
DEBUG(dbgs() << "Top level segment at " << Line << ":" << Col << "\n");
@@ -282,20 +291,17 @@ class SegmentBuilder {
/// Start a segment with the given Region's count.
void startSegment(unsigned Line, unsigned Col, bool IsRegionEntry,
const CountedRegion &Region) {
- if (Segments.empty())
- Segments.emplace_back(Line, Col, IsRegionEntry);
- CoverageSegment S = Segments.back();
// Avoid creating empty regions.
- if (S.Line != Line || S.Col != Col) {
- Segments.emplace_back(Line, Col, IsRegionEntry);
- S = Segments.back();
- }
+ if (!Segments.empty() && Segments.back().Line == Line &&
+ Segments.back().Col == Col)
+ Segments.pop_back();
DEBUG(dbgs() << "Segment at " << Line << ":" << Col);
// Set this region's count.
if (Region.Kind != coverage::CounterMappingRegion::SkippedRegion) {
DEBUG(dbgs() << " with count " << Region.ExecutionCount);
- Segments.back().setCount(Region.ExecutionCount);
- }
+ Segments.emplace_back(Line, Col, Region.ExecutionCount, IsRegionEntry);
+ } else
+ Segments.emplace_back(Line, Col, IsRegionEntry);
DEBUG(dbgs() << "\n");
}
@@ -316,29 +322,89 @@ class SegmentBuilder {
startSegment(Line, Col, false, *ActiveRegions.back());
}
-public:
- /// Build a list of CoverageSegments from a sorted list of Regions.
- std::vector<CoverageSegment> buildSegments(ArrayRef<CountedRegion> Regions) {
- const CountedRegion *PrevRegion = nullptr;
+ void buildSegmentsImpl(ArrayRef<CountedRegion> Regions) {
for (const auto &Region : Regions) {
// Pop any regions that end before this one starts.
while (!ActiveRegions.empty() &&
ActiveRegions.back()->endLoc() <= Region.startLoc())
popRegion();
- if (PrevRegion && PrevRegion->startLoc() == Region.startLoc() &&
- PrevRegion->endLoc() == Region.endLoc()) {
- if (Region.Kind == coverage::CounterMappingRegion::CodeRegion)
- Segments.back().addCount(Region.ExecutionCount);
- } else {
- // Add this region to the stack.
- ActiveRegions.push_back(&Region);
- startSegment(Region);
- }
- PrevRegion = &Region;
+ // Add this region to the stack.
+ ActiveRegions.push_back(&Region);
+ startSegment(Region);
}
// Pop any regions that are left in the stack.
while (!ActiveRegions.empty())
popRegion();
+ }
+
+ /// Sort a nested sequence of regions from a single file.
+ static void sortNestedRegions(MutableArrayRef<CountedRegion> Regions) {
+ std::sort(Regions.begin(), Regions.end(), [](const CountedRegion &LHS,
+ const CountedRegion &RHS) {
+ if (LHS.startLoc() != RHS.startLoc())
+ return LHS.startLoc() < RHS.startLoc();
+ if (LHS.endLoc() != RHS.endLoc())
+ // When LHS completely contains RHS, we sort LHS first.
+ return RHS.endLoc() < LHS.endLoc();
+ // If LHS and RHS cover the same area, we need to sort them according
+ // to their kinds so that the most suitable region will become "active"
+ // in combineRegions(). Because we accumulate counter values only from
+ // regions of the same kind as the first region of the area, prefer
+ // CodeRegion to ExpansionRegion and ExpansionRegion to SkippedRegion.
+ static_assert(coverage::CounterMappingRegion::CodeRegion <
+ coverage::CounterMappingRegion::ExpansionRegion &&
+ coverage::CounterMappingRegion::ExpansionRegion <
+ coverage::CounterMappingRegion::SkippedRegion,
+ "Unexpected order of region kind values");
+ return LHS.Kind < RHS.Kind;
+ });
+ }
+
+ /// Combine counts of regions which cover the same area.
+ static ArrayRef<CountedRegion>
+ combineRegions(MutableArrayRef<CountedRegion> Regions) {
+ if (Regions.empty())
+ return Regions;
+ auto Active = Regions.begin();
+ auto End = Regions.end();
+ for (auto I = Regions.begin() + 1; I != End; ++I) {
+ if (Active->startLoc() != I->startLoc() ||
+ Active->endLoc() != I->endLoc()) {
+ // Shift to the next region.
+ ++Active;
+ if (Active != I)
+ *Active = *I;
+ continue;
+ }
+ // Merge duplicate region.
+ // If CodeRegions and ExpansionRegions cover the same area, it's probably
+ // a macro which is fully expanded to another macro. In that case, we need
+ // to accumulate counts only from CodeRegions, or else the area will be
+ // counted twice.
+ // On the other hand, a macro may have a nested macro in its body. If the
+ // outer macro is used several times, the ExpansionRegion for the nested
+ // macro will also be added several times. These ExpansionRegions cover
+ // the same source locations and have to be combined to reach the correct
+ // value for that area.
+ // We add counts of the regions of the same kind as the active region
+ // to handle the both situations.
+ if (I->Kind == Active->Kind)
+ Active->ExecutionCount += I->ExecutionCount;
+ }
+ return Regions.drop_back(std::distance(++Active, End));
+ }
+
+public:
+ /// Build a list of CoverageSegments from a list of Regions.
+ static std::vector<CoverageSegment>
+ buildSegments(MutableArrayRef<CountedRegion> Regions) {
+ std::vector<CoverageSegment> Segments;
+ SegmentBuilder Builder(Segments);
+
+ sortNestedRegions(Regions);
+ ArrayRef<CountedRegion> CombinedRegions = combineRegions(Regions);
+
+ Builder.buildSegmentsImpl(CombinedRegions);
return Segments;
}
};
@@ -364,21 +430,7 @@ static SmallBitVector gatherFileIDs(StringRef SourceFile,
return FilenameEquivalence;
}
-static Optional<unsigned> findMainViewFileID(StringRef SourceFile,
- const FunctionRecord &Function) {
- SmallBitVector IsNotExpandedFile(Function.Filenames.size(), true);
- SmallBitVector FilenameEquivalence = gatherFileIDs(SourceFile, Function);
- for (const auto &CR : Function.CountedRegions)
- if (CR.Kind == CounterMappingRegion::ExpansionRegion &&
- FilenameEquivalence[CR.FileID])
- IsNotExpandedFile[CR.ExpandedFileID] = false;
- IsNotExpandedFile &= FilenameEquivalence;
- int I = IsNotExpandedFile.find_first();
- if (I == -1)
- return None;
- return I;
-}
-
+/// Return the ID of the file where the definition of the function is located.
static Optional<unsigned> findMainViewFileID(const FunctionRecord &Function) {
SmallBitVector IsNotExpandedFile(Function.Filenames.size(), true);
for (const auto &CR : Function.CountedRegions)
@@ -390,47 +442,43 @@ static Optional<unsigned> findMainViewFileID(const FunctionRecord &Function) {
return I;
}
-/// Sort a nested sequence of regions from a single file.
-template <class It> static void sortNestedRegions(It First, It Last) {
- std::sort(First, Last,
- [](const CountedRegion &LHS, const CountedRegion &RHS) {
- if (LHS.startLoc() == RHS.startLoc())
- // When LHS completely contains RHS, we sort LHS first.
- return RHS.endLoc() < LHS.endLoc();
- return LHS.startLoc() < RHS.startLoc();
- });
+/// Check if SourceFile is the file that contains the definition of
+/// the Function. Return the ID of the file in that case or None otherwise.
+static Optional<unsigned> findMainViewFileID(StringRef SourceFile,
+ const FunctionRecord &Function) {
+ Optional<unsigned> I = findMainViewFileID(Function);
+ if (I && SourceFile == Function.Filenames[*I])
+ return I;
+ return None;
}
static bool isExpansion(const CountedRegion &R, unsigned FileID) {
return R.Kind == CounterMappingRegion::ExpansionRegion && R.FileID == FileID;
}
-CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) {
+CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const {
CoverageData FileCoverage(Filename);
std::vector<coverage::CountedRegion> Regions;
for (const auto &Function : Functions) {
auto MainFileID = findMainViewFileID(Filename, Function);
- if (!MainFileID)
- continue;
auto FileIDs = gatherFileIDs(Filename, Function);
for (const auto &CR : Function.CountedRegions)
if (FileIDs.test(CR.FileID)) {
Regions.push_back(CR);
- if (isExpansion(CR, *MainFileID))
+ if (MainFileID && isExpansion(CR, *MainFileID))
FileCoverage.Expansions.emplace_back(CR, Function);
}
}
- sortNestedRegions(Regions.begin(), Regions.end());
DEBUG(dbgs() << "Emitting segments for file: " << Filename << "\n");
- FileCoverage.Segments = SegmentBuilder().buildSegments(Regions);
+ FileCoverage.Segments = SegmentBuilder::buildSegments(Regions);
return FileCoverage;
}
std::vector<const FunctionRecord *>
-CoverageMapping::getInstantiations(StringRef Filename) {
+CoverageMapping::getInstantiations(StringRef Filename) const {
FunctionInstantiationSetCollector InstantiationSetCollector;
for (const auto &Function : Functions) {
auto MainFileID = findMainViewFileID(Filename, Function);
@@ -450,7 +498,7 @@ CoverageMapping::getInstantiations(StringRef Filename) {
}
CoverageData
-CoverageMapping::getCoverageForFunction(const FunctionRecord &Function) {
+CoverageMapping::getCoverageForFunction(const FunctionRecord &Function) const {
auto MainFileID = findMainViewFileID(Function);
if (!MainFileID)
return CoverageData();
@@ -464,15 +512,14 @@ CoverageMapping::getCoverageForFunction(const FunctionRecord &Function) {
FunctionCoverage.Expansions.emplace_back(CR, Function);
}
- sortNestedRegions(Regions.begin(), Regions.end());
DEBUG(dbgs() << "Emitting segments for function: " << Function.Name << "\n");
- FunctionCoverage.Segments = SegmentBuilder().buildSegments(Regions);
+ FunctionCoverage.Segments = SegmentBuilder::buildSegments(Regions);
return FunctionCoverage;
}
-CoverageData
-CoverageMapping::getCoverageForExpansion(const ExpansionRecord &Expansion) {
+CoverageData CoverageMapping::getCoverageForExpansion(
+ const ExpansionRecord &Expansion) const {
CoverageData ExpansionCoverage(
Expansion.Function.Filenames[Expansion.FileID]);
std::vector<coverage::CountedRegion> Regions;
@@ -483,36 +530,45 @@ CoverageMapping::getCoverageForExpansion(const ExpansionRecord &Expansion) {
ExpansionCoverage.Expansions.emplace_back(CR, Expansion.Function);
}
- sortNestedRegions(Regions.begin(), Regions.end());
DEBUG(dbgs() << "Emitting segments for expansion of file " << Expansion.FileID
<< "\n");
- ExpansionCoverage.Segments = SegmentBuilder().buildSegments(Regions);
+ ExpansionCoverage.Segments = SegmentBuilder::buildSegments(Regions);
return ExpansionCoverage;
}
namespace {
+std::string getCoverageMapErrString(coveragemap_error Err) {
+ switch (Err) {
+ case coveragemap_error::success:
+ return "Success";
+ case coveragemap_error::eof:
+ return "End of File";
+ case coveragemap_error::no_data_found:
+ return "No coverage data found";
+ case coveragemap_error::unsupported_version:
+ return "Unsupported coverage format version";
+ case coveragemap_error::truncated:
+ return "Truncated coverage data";
+ case coveragemap_error::malformed:
+ return "Malformed coverage data";
+ }
+ llvm_unreachable("A value of coveragemap_error has no message.");
+}
+
+// FIXME: This class is only here to support the transition to llvm::Error. It
+// will be removed once this transition is complete. Clients should prefer to
+// deal with the Error value directly, rather than converting to error_code.
class CoverageMappingErrorCategoryType : public std::error_category {
const char *name() const LLVM_NOEXCEPT override { return "llvm.coveragemap"; }
std::string message(int IE) const override {
- auto E = static_cast<coveragemap_error>(IE);
- switch (E) {
- case coveragemap_error::success:
- return "Success";
- case coveragemap_error::eof:
- return "End of File";
- case coveragemap_error::no_data_found:
- return "No coverage data found";
- case coveragemap_error::unsupported_version:
- return "Unsupported coverage format version";
- case coveragemap_error::truncated:
- return "Truncated coverage data";
- case coveragemap_error::malformed:
- return "Malformed coverage data";
- }
- llvm_unreachable("A value of coveragemap_error has no message.");
+ return getCoverageMapErrString(static_cast<coveragemap_error>(IE));
}
};
+} // end anonymous namespace
+
+std::string CoverageMapError::message() const {
+ return getCoverageMapErrString(Err);
}
static ManagedStatic<CoverageMappingErrorCategoryType> ErrorCategory;
@@ -520,3 +576,5 @@ static ManagedStatic<CoverageMappingErrorCategoryType> ErrorCategory;
const std::error_category &llvm::coverage::coveragemap_category() {
return *ErrorCategory;
}
+
+char CoverageMapError::ID = 0;
diff --git a/lib/ProfileData/CoverageMappingReader.cpp b/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index 89e1cf42c577..1a4b4f590841 100644
--- a/lib/ProfileData/CoverageMappingReader.cpp
+++ b/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -12,8 +12,8 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/ProfileData/CoverageMappingReader.h"
-#include "llvm/ADT/DenseSet.h"
+#include "llvm/ProfileData/Coverage/CoverageMappingReader.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/Object/MachOUniversal.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/Debug.h"
@@ -31,49 +31,54 @@ using namespace object;
void CoverageMappingIterator::increment() {
// Check if all the records were read or if an error occurred while reading
// the next record.
- if (Reader->readNextRecord(Record))
- *this = CoverageMappingIterator();
+ if (auto E = Reader->readNextRecord(Record)) {
+ handleAllErrors(std::move(E), [&](const CoverageMapError &CME) {
+ if (CME.get() == coveragemap_error::eof)
+ *this = CoverageMappingIterator();
+ else
+ llvm_unreachable("Unexpected error in coverage mapping iterator");
+ });
+ }
}
-std::error_code RawCoverageReader::readULEB128(uint64_t &Result) {
+Error RawCoverageReader::readULEB128(uint64_t &Result) {
if (Data.size() < 1)
- return coveragemap_error::truncated;
+ return make_error<CoverageMapError>(coveragemap_error::truncated);
unsigned N = 0;
Result = decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N);
if (N > Data.size())
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
Data = Data.substr(N);
- return std::error_code();
+ return Error::success();
}
-std::error_code RawCoverageReader::readIntMax(uint64_t &Result,
- uint64_t MaxPlus1) {
+Error RawCoverageReader::readIntMax(uint64_t &Result, uint64_t MaxPlus1) {
if (auto Err = readULEB128(Result))
return Err;
if (Result >= MaxPlus1)
- return coveragemap_error::malformed;
- return std::error_code();
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
+ return Error::success();
}
-std::error_code RawCoverageReader::readSize(uint64_t &Result) {
+Error RawCoverageReader::readSize(uint64_t &Result) {
if (auto Err = readULEB128(Result))
return Err;
// Sanity check the number.
if (Result > Data.size())
- return coveragemap_error::malformed;
- return std::error_code();
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
+ return Error::success();
}
-std::error_code RawCoverageReader::readString(StringRef &Result) {
+Error RawCoverageReader::readString(StringRef &Result) {
uint64_t Length;
if (auto Err = readSize(Length))
return Err;
Result = Data.substr(0, Length);
Data = Data.substr(Length);
- return std::error_code();
+ return Error::success();
}
-std::error_code RawCoverageFilenamesReader::read() {
+Error RawCoverageFilenamesReader::read() {
uint64_t NumFilenames;
if (auto Err = readSize(NumFilenames))
return Err;
@@ -83,19 +88,18 @@ std::error_code RawCoverageFilenamesReader::read() {
return Err;
Filenames.push_back(Filename);
}
- return std::error_code();
+ return Error::success();
}
-std::error_code RawCoverageMappingReader::decodeCounter(unsigned Value,
- Counter &C) {
+Error RawCoverageMappingReader::decodeCounter(unsigned Value, Counter &C) {
auto Tag = Value & Counter::EncodingTagMask;
switch (Tag) {
case Counter::Zero:
C = Counter::getZero();
- return std::error_code();
+ return Error::success();
case Counter::CounterValueReference:
C = Counter::getCounter(Value >> Counter::EncodingTagBits);
- return std::error_code();
+ return Error::success();
default:
break;
}
@@ -105,25 +109,25 @@ std::error_code RawCoverageMappingReader::decodeCounter(unsigned Value,
case CounterExpression::Add: {
auto ID = Value >> Counter::EncodingTagBits;
if (ID >= Expressions.size())
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
Expressions[ID].Kind = CounterExpression::ExprKind(Tag);
C = Counter::getExpression(ID);
break;
}
default:
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
}
- return std::error_code();
+ return Error::success();
}
-std::error_code RawCoverageMappingReader::readCounter(Counter &C) {
+Error RawCoverageMappingReader::readCounter(Counter &C) {
uint64_t EncodedCounter;
if (auto Err =
readIntMax(EncodedCounter, std::numeric_limits<unsigned>::max()))
return Err;
if (auto Err = decodeCounter(EncodedCounter, C))
return Err;
- return std::error_code();
+ return Error::success();
}
static const unsigned EncodingExpansionRegionBit = 1
@@ -132,7 +136,7 @@ static const unsigned EncodingExpansionRegionBit = 1
/// \brief Read the sub-array of regions for the given inferred file id.
/// \param NumFileIDs the number of file ids that are defined for this
/// function.
-std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
+Error RawCoverageMappingReader::readMappingRegionsSubArray(
std::vector<CounterMappingRegion> &MappingRegions, unsigned InferredFileID,
size_t NumFileIDs) {
uint64_t NumRegions;
@@ -160,7 +164,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
ExpandedFileID = EncodedCounterAndRegion >>
Counter::EncodingCounterTagAndExpansionRegionTagBits;
if (ExpandedFileID >= NumFileIDs)
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
} else {
switch (EncodedCounterAndRegion >>
Counter::EncodingCounterTagAndExpansionRegionTagBits) {
@@ -171,7 +175,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
Kind = CounterMappingRegion::SkippedRegion;
break;
default:
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
}
}
}
@@ -184,7 +188,7 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
if (auto Err = readULEB128(ColumnStart))
return Err;
if (ColumnStart > std::numeric_limits<unsigned>::max())
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
if (auto Err = readIntMax(NumLines, std::numeric_limits<unsigned>::max()))
return Err;
if (auto Err = readIntMax(ColumnEnd, std::numeric_limits<unsigned>::max()))
@@ -218,10 +222,10 @@ std::error_code RawCoverageMappingReader::readMappingRegionsSubArray(
C, InferredFileID, ExpandedFileID, LineStart, ColumnStart,
LineStart + NumLines, ColumnEnd, Kind));
}
- return std::error_code();
+ return Error::success();
}
-std::error_code RawCoverageMappingReader::read() {
+Error RawCoverageMappingReader::read() {
// Read the virtual file mapping.
llvm::SmallVector<unsigned, 8> VirtualFileMapping;
@@ -287,14 +291,44 @@ std::error_code RawCoverageMappingReader::read() {
}
}
- return std::error_code();
+ return Error::success();
}
-std::error_code InstrProfSymtab::create(SectionRef &Section) {
- if (auto Err = Section.getContents(Data))
- return Err;
+Expected<bool> RawCoverageMappingDummyChecker::isDummy() {
+ // A dummy coverage mapping data consists of just one region with zero count.
+ uint64_t NumFileMappings;
+ if (Error Err = readSize(NumFileMappings))
+ return std::move(Err);
+ if (NumFileMappings != 1)
+ return false;
+ // We don't expect any specific value for the filename index, just skip it.
+ uint64_t FilenameIndex;
+ if (Error Err =
+ readIntMax(FilenameIndex, std::numeric_limits<unsigned>::max()))
+ return std::move(Err);
+ uint64_t NumExpressions;
+ if (Error Err = readSize(NumExpressions))
+ return std::move(Err);
+ if (NumExpressions != 0)
+ return false;
+ uint64_t NumRegions;
+ if (Error Err = readSize(NumRegions))
+ return std::move(Err);
+ if (NumRegions != 1)
+ return false;
+ uint64_t EncodedCounterAndRegion;
+ if (Error Err = readIntMax(EncodedCounterAndRegion,
+ std::numeric_limits<unsigned>::max()))
+ return std::move(Err);
+ unsigned Tag = EncodedCounterAndRegion & Counter::EncodingTagMask;
+ return Tag == Counter::Zero;
+}
+
+Error InstrProfSymtab::create(SectionRef &Section) {
+ if (auto EC = Section.getContents(Data))
+ return errorCodeToError(EC);
Address = Section.getAddress();
- return std::error_code();
+ return Error::success();
}
StringRef InstrProfSymtab::getFuncName(uint64_t Pointer, size_t Size) {
@@ -306,40 +340,124 @@ StringRef InstrProfSymtab::getFuncName(uint64_t Pointer, size_t Size) {
return Data.substr(Pointer - Address, Size);
}
-template <typename T, support::endianness Endian>
-static std::error_code readCoverageMappingData(
- InstrProfSymtab &ProfileNames, StringRef Data,
- std::vector<BinaryCoverageReader::ProfileMappingRecord> &Records,
- std::vector<StringRef> &Filenames) {
- using namespace support;
- llvm::DenseSet<T> UniqueFunctionMappingData;
+// Check if the mapping data is a dummy, i.e. is emitted for an unused function.
+static Expected<bool> isCoverageMappingDummy(uint64_t Hash, StringRef Mapping) {
+ // The hash value of dummy mapping records is always zero.
+ if (Hash)
+ return false;
+ return RawCoverageMappingDummyChecker(Mapping).isDummy();
+}
- // Read the records in the coverage data section.
- for (const char *Buf = Data.data(), *End = Buf + Data.size(); Buf < End;) {
+namespace {
+struct CovMapFuncRecordReader {
+ // The interface to read coverage mapping function records for a module.
+ //
+ // \p Buf points to the buffer containing the \c CovHeader of the coverage
+ // mapping data associated with the module.
+ //
+ // Returns a pointer to the next \c CovHeader if it exists, or a pointer
+ // greater than \p End if not.
+ virtual Expected<const char *> readFunctionRecords(const char *Buf,
+ const char *End) = 0;
+ virtual ~CovMapFuncRecordReader() {}
+ template <class IntPtrT, support::endianness Endian>
+ static Expected<std::unique_ptr<CovMapFuncRecordReader>>
+ get(coverage::CovMapVersion Version, InstrProfSymtab &P,
+ std::vector<BinaryCoverageReader::ProfileMappingRecord> &R,
+ std::vector<StringRef> &F);
+};
+
+// A class for reading coverage mapping function records for a module.
+template <coverage::CovMapVersion Version, class IntPtrT,
+ support::endianness Endian>
+class VersionedCovMapFuncRecordReader : public CovMapFuncRecordReader {
+ typedef typename coverage::CovMapTraits<
+ Version, IntPtrT>::CovMapFuncRecordType FuncRecordType;
+ typedef typename coverage::CovMapTraits<Version, IntPtrT>::NameRefType
+ NameRefType;
+
+ // Maps function's name references to the indexes of their records
+ // in \c Records.
+ llvm::DenseMap<NameRefType, size_t> FunctionRecords;
+ InstrProfSymtab &ProfileNames;
+ std::vector<StringRef> &Filenames;
+ std::vector<BinaryCoverageReader::ProfileMappingRecord> &Records;
+
+ // Add the record to the collection if we don't already have a record that
+ // points to the same function name. This is useful to ignore the redundant
+ // records for the functions with ODR linkage.
+ // In addition, prefer records with real coverage mapping data to dummy
+ // records, which were emitted for inline functions which were seen but
+ // not used in the corresponding translation unit.
+ Error insertFunctionRecordIfNeeded(const FuncRecordType *CFR,
+ StringRef Mapping, size_t FilenamesBegin) {
+ uint64_t FuncHash = CFR->template getFuncHash<Endian>();
+ NameRefType NameRef = CFR->template getFuncNameRef<Endian>();
+ auto InsertResult =
+ FunctionRecords.insert(std::make_pair(NameRef, Records.size()));
+ if (InsertResult.second) {
+ StringRef FuncName;
+ if (Error Err = CFR->template getFuncName<Endian>(ProfileNames, FuncName))
+ return Err;
+ Records.emplace_back(Version, FuncName, FuncHash, Mapping, FilenamesBegin,
+ Filenames.size() - FilenamesBegin);
+ return Error::success();
+ }
+ // Update the existing record if it's a dummy and the new record is real.
+ size_t OldRecordIndex = InsertResult.first->second;
+ BinaryCoverageReader::ProfileMappingRecord &OldRecord =
+ Records[OldRecordIndex];
+ Expected<bool> OldIsDummyExpected = isCoverageMappingDummy(
+ OldRecord.FunctionHash, OldRecord.CoverageMapping);
+ if (Error Err = OldIsDummyExpected.takeError())
+ return Err;
+ if (!*OldIsDummyExpected)
+ return Error::success();
+ Expected<bool> NewIsDummyExpected =
+ isCoverageMappingDummy(FuncHash, Mapping);
+ if (Error Err = NewIsDummyExpected.takeError())
+ return Err;
+ if (*NewIsDummyExpected)
+ return Error::success();
+ OldRecord.FunctionHash = FuncHash;
+ OldRecord.CoverageMapping = Mapping;
+ OldRecord.FilenamesBegin = FilenamesBegin;
+ OldRecord.FilenamesSize = Filenames.size() - FilenamesBegin;
+ return Error::success();
+ }
+
+public:
+ VersionedCovMapFuncRecordReader(
+ InstrProfSymtab &P,
+ std::vector<BinaryCoverageReader::ProfileMappingRecord> &R,
+ std::vector<StringRef> &F)
+ : ProfileNames(P), Filenames(F), Records(R) {}
+ ~VersionedCovMapFuncRecordReader() override {}
+
+ Expected<const char *> readFunctionRecords(const char *Buf,
+ const char *End) override {
+ using namespace support;
if (Buf + sizeof(CovMapHeader) > End)
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
auto CovHeader = reinterpret_cast<const coverage::CovMapHeader *>(Buf);
uint32_t NRecords = CovHeader->getNRecords<Endian>();
uint32_t FilenamesSize = CovHeader->getFilenamesSize<Endian>();
uint32_t CoverageSize = CovHeader->getCoverageSize<Endian>();
- uint32_t Version = CovHeader->getVersion<Endian>();
- Buf = reinterpret_cast<const char *>(++CovHeader);
-
- if (Version > coverage::CoverageMappingCurrentVersion)
- return coveragemap_error::unsupported_version;
+ assert((CovMapVersion)CovHeader->getVersion<Endian>() == Version);
+ Buf = reinterpret_cast<const char *>(CovHeader + 1);
// Skip past the function records, saving the start and end for later.
const char *FunBuf = Buf;
- Buf += NRecords * sizeof(coverage::CovMapFunctionRecord<T>);
+ Buf += NRecords * sizeof(FuncRecordType);
const char *FunEnd = Buf;
// Get the filenames.
if (Buf + FilenamesSize > End)
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
size_t FilenamesBegin = Filenames.size();
RawCoverageFilenamesReader Reader(StringRef(Buf, FilenamesSize), Filenames);
if (auto Err = Reader.read())
- return Err;
+ return std::move(Err);
Buf += FilenamesSize;
// We'll read the coverage mapping records in the loop below.
@@ -348,115 +466,156 @@ static std::error_code readCoverageMappingData(
const char *CovEnd = Buf;
if (Buf > End)
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
// Each coverage map has an alignment of 8, so we need to adjust alignment
// before reading the next map.
Buf += alignmentAdjustment(Buf, 8);
- auto CFR =
- reinterpret_cast<const coverage::CovMapFunctionRecord<T> *>(FunBuf);
+ auto CFR = reinterpret_cast<const FuncRecordType *>(FunBuf);
while ((const char *)CFR < FunEnd) {
// Read the function information
uint32_t DataSize = CFR->template getDataSize<Endian>();
- uint64_t FuncHash = CFR->template getFuncHash<Endian>();
// Now use that to read the coverage data.
if (CovBuf + DataSize > CovEnd)
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
auto Mapping = StringRef(CovBuf, DataSize);
CovBuf += DataSize;
- // Ignore this record if we already have a record that points to the same
- // function name. This is useful to ignore the redundant records for the
- // functions with ODR linkage.
- T NameRef = CFR->template getFuncNameRef<Endian>();
- if (!UniqueFunctionMappingData.insert(NameRef).second)
- continue;
-
- StringRef FuncName;
- if (std::error_code EC =
- CFR->template getFuncName<Endian>(ProfileNames, FuncName))
- return EC;
- Records.push_back(BinaryCoverageReader::ProfileMappingRecord(
- CoverageMappingVersion(Version), FuncName, FuncHash, Mapping,
- FilenamesBegin, Filenames.size() - FilenamesBegin));
+ if (Error Err =
+ insertFunctionRecordIfNeeded(CFR, Mapping, FilenamesBegin))
+ return std::move(Err);
CFR++;
}
+ return Buf;
}
+};
+} // end anonymous namespace
- return std::error_code();
+template <class IntPtrT, support::endianness Endian>
+Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get(
+ coverage::CovMapVersion Version, InstrProfSymtab &P,
+ std::vector<BinaryCoverageReader::ProfileMappingRecord> &R,
+ std::vector<StringRef> &F) {
+ using namespace coverage;
+ switch (Version) {
+ case CovMapVersion::Version1:
+ return llvm::make_unique<VersionedCovMapFuncRecordReader<
+ CovMapVersion::Version1, IntPtrT, Endian>>(P, R, F);
+ case CovMapVersion::Version2:
+ // Decompress the name data.
+ if (Error E = P.create(P.getNameData()))
+ return std::move(E);
+ return llvm::make_unique<VersionedCovMapFuncRecordReader<
+ CovMapVersion::Version2, IntPtrT, Endian>>(P, R, F);
+ }
+ llvm_unreachable("Unsupported version");
}
+template <typename T, support::endianness Endian>
+static Error readCoverageMappingData(
+ InstrProfSymtab &ProfileNames, StringRef Data,
+ std::vector<BinaryCoverageReader::ProfileMappingRecord> &Records,
+ std::vector<StringRef> &Filenames) {
+ using namespace coverage;
+ // Read the records in the coverage data section.
+ auto CovHeader =
+ reinterpret_cast<const coverage::CovMapHeader *>(Data.data());
+ CovMapVersion Version = (CovMapVersion)CovHeader->getVersion<Endian>();
+ if (Version > coverage::CovMapVersion::CurrentVersion)
+ return make_error<CoverageMapError>(coveragemap_error::unsupported_version);
+ Expected<std::unique_ptr<CovMapFuncRecordReader>> ReaderExpected =
+ CovMapFuncRecordReader::get<T, Endian>(Version, ProfileNames, Records,
+ Filenames);
+ if (Error E = ReaderExpected.takeError())
+ return E;
+ auto Reader = std::move(ReaderExpected.get());
+ for (const char *Buf = Data.data(), *End = Buf + Data.size(); Buf < End;) {
+ auto NextHeaderOrErr = Reader->readFunctionRecords(Buf, End);
+ if (auto E = NextHeaderOrErr.takeError())
+ return E;
+ Buf = NextHeaderOrErr.get();
+ }
+ return Error::success();
+}
static const char *TestingFormatMagic = "llvmcovmtestdata";
-static std::error_code loadTestingFormat(StringRef Data,
- InstrProfSymtab &ProfileNames,
- StringRef &CoverageMapping,
- uint8_t &BytesInAddress,
- support::endianness &Endian) {
+static Error loadTestingFormat(StringRef Data, InstrProfSymtab &ProfileNames,
+ StringRef &CoverageMapping,
+ uint8_t &BytesInAddress,
+ support::endianness &Endian) {
BytesInAddress = 8;
Endian = support::endianness::little;
Data = Data.substr(StringRef(TestingFormatMagic).size());
if (Data.size() < 1)
- return coveragemap_error::truncated;
+ return make_error<CoverageMapError>(coveragemap_error::truncated);
unsigned N = 0;
auto ProfileNamesSize =
decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N);
if (N > Data.size())
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
Data = Data.substr(N);
if (Data.size() < 1)
- return coveragemap_error::truncated;
+ return make_error<CoverageMapError>(coveragemap_error::truncated);
N = 0;
uint64_t Address =
decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N);
if (N > Data.size())
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
Data = Data.substr(N);
if (Data.size() < ProfileNamesSize)
- return coveragemap_error::malformed;
- ProfileNames.create(Data.substr(0, ProfileNamesSize), Address);
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
+ if (Error E = ProfileNames.create(Data.substr(0, ProfileNamesSize), Address))
+ return E;
CoverageMapping = Data.substr(ProfileNamesSize);
- return std::error_code();
+ // Skip the padding bytes because coverage map data has an alignment of 8.
+ if (CoverageMapping.size() < 1)
+ return make_error<CoverageMapError>(coveragemap_error::truncated);
+ size_t Pad = alignmentAdjustment(CoverageMapping.data(), 8);
+ if (CoverageMapping.size() < Pad)
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
+ CoverageMapping = CoverageMapping.substr(Pad);
+ return Error::success();
}
-static ErrorOr<SectionRef> lookupSection(ObjectFile &OF, StringRef Name) {
+static Expected<SectionRef> lookupSection(ObjectFile &OF, StringRef Name) {
StringRef FoundName;
for (const auto &Section : OF.sections()) {
if (auto EC = Section.getName(FoundName))
- return EC;
+ return errorCodeToError(EC);
if (FoundName == Name)
return Section;
}
- return coveragemap_error::no_data_found;
+ return make_error<CoverageMapError>(coveragemap_error::no_data_found);
}
-static std::error_code
-loadBinaryFormat(MemoryBufferRef ObjectBuffer, InstrProfSymtab &ProfileNames,
- StringRef &CoverageMapping, uint8_t &BytesInAddress,
- support::endianness &Endian, StringRef Arch) {
+static Error loadBinaryFormat(MemoryBufferRef ObjectBuffer,
+ InstrProfSymtab &ProfileNames,
+ StringRef &CoverageMapping,
+ uint8_t &BytesInAddress,
+ support::endianness &Endian, StringRef Arch) {
auto BinOrErr = object::createBinary(ObjectBuffer);
- if (std::error_code EC = BinOrErr.getError())
- return EC;
+ if (!BinOrErr)
+ return BinOrErr.takeError();
auto Bin = std::move(BinOrErr.get());
std::unique_ptr<ObjectFile> OF;
if (auto *Universal = dyn_cast<object::MachOUniversalBinary>(Bin.get())) {
// If we have a universal binary, try to look up the object for the
// appropriate architecture.
auto ObjectFileOrErr = Universal->getObjectForArch(Arch);
- if (std::error_code EC = ObjectFileOrErr.getError())
- return EC;
+ if (!ObjectFileOrErr)
+ return ObjectFileOrErr.takeError();
OF = std::move(ObjectFileOrErr.get());
} else if (isa<object::ObjectFile>(Bin.get())) {
// For any other object file, upcast and take ownership.
OF.reset(cast<object::ObjectFile>(Bin.release()));
// If we've asked for a particular arch, make sure they match.
if (!Arch.empty() && OF->getArch() != Triple(Arch).getArch())
- return object_error::arch_not_found;
+ return errorCodeToError(object_error::arch_not_found);
} else
// We can only handle object files.
- return coveragemap_error::malformed;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
// The coverage uses native pointer sizes for the object it's written in.
BytesInAddress = OF->getBytesInAddress();
@@ -465,65 +624,68 @@ loadBinaryFormat(MemoryBufferRef ObjectBuffer, InstrProfSymtab &ProfileNames,
// Look for the sections that we are interested in.
auto NamesSection = lookupSection(*OF, getInstrProfNameSectionName(false));
- if (auto EC = NamesSection.getError())
- return EC;
+ if (auto E = NamesSection.takeError())
+ return E;
auto CoverageSection =
lookupSection(*OF, getInstrProfCoverageSectionName(false));
- if (auto EC = CoverageSection.getError())
- return EC;
+ if (auto E = CoverageSection.takeError())
+ return E;
// Get the contents of the given sections.
- if (std::error_code EC = CoverageSection->getContents(CoverageMapping))
- return EC;
- if (std::error_code EC = ProfileNames.create(*NamesSection))
- return EC;
+ if (auto EC = CoverageSection->getContents(CoverageMapping))
+ return errorCodeToError(EC);
+ if (Error E = ProfileNames.create(*NamesSection))
+ return E;
- return std::error_code();
+ return Error::success();
}
-ErrorOr<std::unique_ptr<BinaryCoverageReader>>
+Expected<std::unique_ptr<BinaryCoverageReader>>
BinaryCoverageReader::create(std::unique_ptr<MemoryBuffer> &ObjectBuffer,
StringRef Arch) {
std::unique_ptr<BinaryCoverageReader> Reader(new BinaryCoverageReader());
- InstrProfSymtab ProfileNames;
StringRef Coverage;
uint8_t BytesInAddress;
support::endianness Endian;
- std::error_code EC;
+ Error E;
+ consumeError(std::move(E));
if (ObjectBuffer->getBuffer().startswith(TestingFormatMagic))
// This is a special format used for testing.
- EC = loadTestingFormat(ObjectBuffer->getBuffer(), ProfileNames, Coverage,
- BytesInAddress, Endian);
+ E = loadTestingFormat(ObjectBuffer->getBuffer(), Reader->ProfileNames,
+ Coverage, BytesInAddress, Endian);
else
- EC = loadBinaryFormat(ObjectBuffer->getMemBufferRef(), ProfileNames,
- Coverage, BytesInAddress, Endian, Arch);
- if (EC)
- return EC;
+ E = loadBinaryFormat(ObjectBuffer->getMemBufferRef(), Reader->ProfileNames,
+ Coverage, BytesInAddress, Endian, Arch);
+ if (E)
+ return std::move(E);
if (BytesInAddress == 4 && Endian == support::endianness::little)
- EC = readCoverageMappingData<uint32_t, support::endianness::little>(
- ProfileNames, Coverage, Reader->MappingRecords, Reader->Filenames);
+ E = readCoverageMappingData<uint32_t, support::endianness::little>(
+ Reader->ProfileNames, Coverage, Reader->MappingRecords,
+ Reader->Filenames);
else if (BytesInAddress == 4 && Endian == support::endianness::big)
- EC = readCoverageMappingData<uint32_t, support::endianness::big>(
- ProfileNames, Coverage, Reader->MappingRecords, Reader->Filenames);
+ E = readCoverageMappingData<uint32_t, support::endianness::big>(
+ Reader->ProfileNames, Coverage, Reader->MappingRecords,
+ Reader->Filenames);
else if (BytesInAddress == 8 && Endian == support::endianness::little)
- EC = readCoverageMappingData<uint64_t, support::endianness::little>(
- ProfileNames, Coverage, Reader->MappingRecords, Reader->Filenames);
+ E = readCoverageMappingData<uint64_t, support::endianness::little>(
+ Reader->ProfileNames, Coverage, Reader->MappingRecords,
+ Reader->Filenames);
else if (BytesInAddress == 8 && Endian == support::endianness::big)
- EC = readCoverageMappingData<uint64_t, support::endianness::big>(
- ProfileNames, Coverage, Reader->MappingRecords, Reader->Filenames);
+ E = readCoverageMappingData<uint64_t, support::endianness::big>(
+ Reader->ProfileNames, Coverage, Reader->MappingRecords,
+ Reader->Filenames);
else
- return coveragemap_error::malformed;
- if (EC)
- return EC;
+ return make_error<CoverageMapError>(coveragemap_error::malformed);
+ if (E)
+ return std::move(E);
return std::move(Reader);
}
-std::error_code
-BinaryCoverageReader::readNextRecord(CoverageMappingRecord &Record) {
+Error BinaryCoverageReader::readNextRecord(CoverageMappingRecord &Record) {
if (CurrentRecord >= MappingRecords.size())
- return coveragemap_error::eof;
+ return make_error<CoverageMapError>(coveragemap_error::eof);
FunctionsFilenames.clear();
Expressions.clear();
@@ -543,5 +705,5 @@ BinaryCoverageReader::readNextRecord(CoverageMappingRecord &Record) {
Record.MappingRegions = MappingRegions;
++CurrentRecord;
- return std::error_code();
+ return Error::success();
}
diff --git a/lib/ProfileData/CoverageMappingWriter.cpp b/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index d90d2f565155..8ff90d62cfdc 100644
--- a/lib/ProfileData/CoverageMappingWriter.cpp
+++ b/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -12,7 +12,7 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/ProfileData/CoverageMappingWriter.h"
+#include "llvm/ProfileData/Coverage/CoverageMappingWriter.h"
#include "llvm/Support/LEB128.h"
using namespace llvm;
diff --git a/lib/ProfileData/Coverage/LLVMBuild.txt b/lib/ProfileData/Coverage/LLVMBuild.txt
new file mode 100644
index 000000000000..fc8284b0ef38
--- /dev/null
+++ b/lib/ProfileData/Coverage/LLVMBuild.txt
@@ -0,0 +1,23 @@
+;===- ./lib/ProfileData/Coverage/LLVMBuild.txt -----------------*- Conf -*--===;
+;
+; The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+; http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = Coverage
+parent = ProfileData
+required_libraries = Core Object ProfileData Support
+
diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp
index d6777639abe7..6962f82a5ef5 100644
--- a/lib/ProfileData/InstrProf.cpp
+++ b/lib/ProfileData/InstrProf.cpp
@@ -17,55 +17,72 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Path.h"
using namespace llvm;
+static cl::opt<bool> StaticFuncFullModulePrefix(
+ "static-func-full-module-prefix", cl::init(false),
+ cl::desc("Use full module build paths in the profile counter names for "
+ "static functions."));
+
namespace {
+std::string getInstrProfErrString(instrprof_error Err) {
+ switch (Err) {
+ case instrprof_error::success:
+ return "Success";
+ case instrprof_error::eof:
+ return "End of File";
+ case instrprof_error::unrecognized_format:
+ return "Unrecognized instrumentation profile encoding format";
+ case instrprof_error::bad_magic:
+ return "Invalid instrumentation profile data (bad magic)";
+ case instrprof_error::bad_header:
+ return "Invalid instrumentation profile data (file header is corrupt)";
+ case instrprof_error::unsupported_version:
+ return "Unsupported instrumentation profile format version";
+ case instrprof_error::unsupported_hash_type:
+ return "Unsupported instrumentation profile hash type";
+ case instrprof_error::too_large:
+ return "Too much profile data";
+ case instrprof_error::truncated:
+ return "Truncated profile data";
+ case instrprof_error::malformed:
+ return "Malformed instrumentation profile data";
+ case instrprof_error::unknown_function:
+ return "No profile data available for function";
+ case instrprof_error::hash_mismatch:
+ return "Function control flow change detected (hash mismatch)";
+ case instrprof_error::count_mismatch:
+ return "Function basic block count change detected (counter mismatch)";
+ case instrprof_error::counter_overflow:
+ return "Counter overflow";
+ case instrprof_error::value_site_count_mismatch:
+ return "Function value site count change detected (counter mismatch)";
+ case instrprof_error::compress_failed:
+ return "Failed to compress data (zlib)";
+ case instrprof_error::uncompress_failed:
+ return "Failed to uncompress data (zlib)";
+ }
+ llvm_unreachable("A value of instrprof_error has no message.");
+}
+
+// FIXME: This class is only here to support the transition to llvm::Error. It
+// will be removed once this transition is complete. Clients should prefer to
+// deal with the Error value directly, rather than converting to error_code.
class InstrProfErrorCategoryType : public std::error_category {
const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; }
std::string message(int IE) const override {
- instrprof_error E = static_cast<instrprof_error>(IE);
- switch (E) {
- case instrprof_error::success:
- return "Success";
- case instrprof_error::eof:
- return "End of File";
- case instrprof_error::unrecognized_format:
- return "Unrecognized instrumentation profile encoding format";
- case instrprof_error::bad_magic:
- return "Invalid instrumentation profile data (bad magic)";
- case instrprof_error::bad_header:
- return "Invalid instrumentation profile data (file header is corrupt)";
- case instrprof_error::unsupported_version:
- return "Unsupported instrumentation profile format version";
- case instrprof_error::unsupported_hash_type:
- return "Unsupported instrumentation profile hash type";
- case instrprof_error::too_large:
- return "Too much profile data";
- case instrprof_error::truncated:
- return "Truncated profile data";
- case instrprof_error::malformed:
- return "Malformed instrumentation profile data";
- case instrprof_error::unknown_function:
- return "No profile data available for function";
- case instrprof_error::hash_mismatch:
- return "Function control flow change detected (hash mismatch)";
- case instrprof_error::count_mismatch:
- return "Function basic block count change detected (counter mismatch)";
- case instrprof_error::counter_overflow:
- return "Counter overflow";
- case instrprof_error::value_site_count_mismatch:
- return "Function value site count change detected (counter mismatch)";
- }
- llvm_unreachable("A value of instrprof_error has no message.");
+ return getInstrProfErrString(static_cast<instrprof_error>(IE));
}
};
-}
+} // end anonymous namespace
static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
@@ -75,34 +92,72 @@ const std::error_category &llvm::instrprof_category() {
namespace llvm {
+void SoftInstrProfErrors::addError(instrprof_error IE) {
+ if (IE == instrprof_error::success)
+ return;
+
+ if (FirstError == instrprof_error::success)
+ FirstError = IE;
+
+ switch (IE) {
+ case instrprof_error::hash_mismatch:
+ ++NumHashMismatches;
+ break;
+ case instrprof_error::count_mismatch:
+ ++NumCountMismatches;
+ break;
+ case instrprof_error::counter_overflow:
+ ++NumCounterOverflows;
+ break;
+ case instrprof_error::value_site_count_mismatch:
+ ++NumValueSiteCountMismatches;
+ break;
+ default:
+ llvm_unreachable("Not a soft error");
+ }
+}
+
+std::string InstrProfError::message() const {
+ return getInstrProfErrString(Err);
+}
+
+char InstrProfError::ID = 0;
+
std::string getPGOFuncName(StringRef RawFuncName,
GlobalValue::LinkageTypes Linkage,
StringRef FileName,
uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
+ return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
+}
- // Function names may be prefixed with a binary '1' to indicate
- // that the backend should not modify the symbols due to any platform
- // naming convention. Do not include that '1' in the PGO profile name.
- if (RawFuncName[0] == '\1')
- RawFuncName = RawFuncName.substr(1);
+// Return the PGOFuncName. This function has some special handling when called
+// in LTO optimization. The following only applies when calling in LTO passes
+// (when \c InLTO is true): LTO's internalization privatizes many global linkage
+// symbols. This happens after value profile annotation, but those internal
+// linkage functions should not have a source prefix.
+// To differentiate compiler generated internal symbols from original ones,
+// PGOFuncName meta data are created and attached to the original internal
+// symbols in the value profile annotation step
+// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
+// data, its original linkage must be non-internal.
+std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
+ if (!InLTO) {
+ StringRef FileName = (StaticFuncFullModulePrefix
+ ? F.getParent()->getName()
+ : sys::path::filename(F.getParent()->getName()));
+ return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
+ }
- std::string FuncName = RawFuncName;
- if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
- // For local symbols, prepend the main file name to distinguish them.
- // Do not include the full path in the file name since there's no guarantee
- // that it will stay the same, e.g., if the files are checked out from
- // version control in different locations.
- if (FileName.empty())
- FuncName = FuncName.insert(0, "<unknown>:");
- else
- FuncName = FuncName.insert(0, FileName.str() + ":");
+ // In LTO mode (when InLTO is true), first check if there is a meta data.
+ if (MDNode *MD = getPGOFuncNameMetadata(F)) {
+ StringRef S = cast<MDString>(MD->getOperand(0))->getString();
+ return S.str();
}
- return FuncName;
-}
-std::string getPGOFuncName(const Function &F, uint64_t Version) {
- return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(),
- Version);
+ // If there is no meta data, the function must be a global before the value
+ // profile annotation pass. Its current linkage may be internal if it is
+ // internalized in LTO mode.
+ return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
}
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
@@ -116,8 +171,8 @@ StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
// \p FuncName is the string used as profile lookup key for the function. A
// symbol is created to hold the name. Return the legalized symbol name.
-static std::string getPGOFuncNameVarName(StringRef FuncName,
- GlobalValue::LinkageTypes Linkage) {
+std::string getPGOFuncNameVarName(StringRef FuncName,
+ GlobalValue::LinkageTypes Linkage) {
std::string VarName = getInstrProfNameVarPrefix();
VarName += FuncName;
@@ -125,7 +180,7 @@ static std::string getPGOFuncNameVarName(StringRef FuncName,
return VarName;
// Now fix up illegal chars in local VarName that may upset the assembler.
- const char *InvalidChars = "-:<>\"'";
+ const char *InvalidChars = "-:<>/\"'";
size_t found = VarName.find_first_of(InvalidChars);
while (found != std::string::npos) {
VarName[found] = '_';
@@ -136,7 +191,7 @@ static std::string getPGOFuncNameVarName(StringRef FuncName,
GlobalVariable *createPGOFuncNameVar(Module &M,
GlobalValue::LinkageTypes Linkage,
- StringRef FuncName) {
+ StringRef PGOFuncName) {
// We generally want to match the function's linkage, but available_externally
// and extern_weak both have the wrong semantics, and anything that doesn't
@@ -149,10 +204,11 @@ GlobalVariable *createPGOFuncNameVar(Module &M,
Linkage == GlobalValue::ExternalLinkage)
Linkage = GlobalValue::PrivateLinkage;
- auto *Value = ConstantDataArray::getString(M.getContext(), FuncName, false);
+ auto *Value =
+ ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
auto FuncNameVar =
new GlobalVariable(M, Value->getType(), true, Linkage, Value,
- getPGOFuncNameVarName(FuncName, Linkage));
+ getPGOFuncNameVarName(PGOFuncName, Linkage));
// Hide the symbol so that we correctly get a copy for each executable.
if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
@@ -161,63 +217,83 @@ GlobalVariable *createPGOFuncNameVar(Module &M,
return FuncNameVar;
}
-GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) {
- return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName);
+GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
+ return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
+}
+
+void InstrProfSymtab::create(Module &M, bool InLTO) {
+ for (Function &F : M) {
+ // Function may not have a name: like using asm("") to overwrite the name.
+ // Ignore in this case.
+ if (!F.hasName())
+ continue;
+ const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
+ addFuncName(PGOFuncName);
+ MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
+ }
+
+ finalizeSymtab();
}
-int collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs,
- bool doCompression, std::string &Result) {
+Error collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs,
+ bool doCompression, std::string &Result) {
+ assert(NameStrs.size() && "No name data to emit");
+
uint8_t Header[16], *P = Header;
std::string UncompressedNameStrings =
- join(NameStrs.begin(), NameStrs.end(), StringRef(" "));
+ join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
+
+ assert(StringRef(UncompressedNameStrings)
+ .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
+ "PGO name is invalid (contains separator token)");
unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
P += EncLen;
- auto WriteStringToResult = [&](size_t CompressedLen,
- const std::string &InputStr) {
+ auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
EncLen = encodeULEB128(CompressedLen, P);
P += EncLen;
char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
unsigned HeaderLen = P - &Header[0];
Result.append(HeaderStr, HeaderLen);
Result += InputStr;
- return 0;
+ return Error::success();
};
- if (!doCompression)
+ if (!doCompression) {
return WriteStringToResult(0, UncompressedNameStrings);
+ }
- SmallVector<char, 128> CompressedNameStrings;
+ SmallString<128> CompressedNameStrings;
zlib::Status Success =
zlib::compress(StringRef(UncompressedNameStrings), CompressedNameStrings,
zlib::BestSizeCompression);
if (Success != zlib::StatusOK)
- return 1;
+ return make_error<InstrProfError>(instrprof_error::compress_failed);
- return WriteStringToResult(
- CompressedNameStrings.size(),
- std::string(CompressedNameStrings.data(), CompressedNameStrings.size()));
+ return WriteStringToResult(CompressedNameStrings.size(),
+ CompressedNameStrings);
}
-StringRef getPGOFuncNameInitializer(GlobalVariable *NameVar) {
+StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
StringRef NameStr =
Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
return NameStr;
}
-int collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars,
- std::string &Result) {
+Error collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars,
+ std::string &Result, bool doCompression) {
std::vector<std::string> NameStrs;
for (auto *NameVar : NameVars) {
- NameStrs.push_back(getPGOFuncNameInitializer(NameVar));
+ NameStrs.push_back(getPGOFuncNameVarInitializer(NameVar));
}
- return collectPGOFuncNameStrings(NameStrs, zlib::isAvailable(), Result);
+ return collectPGOFuncNameStrings(
+ NameStrs, zlib::isAvailable() && doCompression, Result);
}
-int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
+Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
const uint8_t *P = reinterpret_cast<const uint8_t *>(NameStrings.data());
const uint8_t *EndP = reinterpret_cast<const uint8_t *>(NameStrings.data() +
NameStrings.size());
@@ -235,7 +311,7 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
CompressedSize);
if (zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
UncompressedSize) != zlib::StatusOK)
- return 1;
+ return make_error<InstrProfError>(instrprof_error::uncompress_failed);
P += CompressedSize;
NameStrings = StringRef(UncompressedNameStrings.data(),
UncompressedNameStrings.size());
@@ -246,7 +322,7 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
}
// Now parse the name strings.
SmallVector<StringRef, 0> Names;
- NameStrings.split(Names, ' ');
+ NameStrings.split(Names, getInstrProfNameSeparator());
for (StringRef &Name : Names)
Symtab.addFuncName(Name);
@@ -254,16 +330,16 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
P++;
}
Symtab.finalizeSymtab();
- return 0;
+ return Error::success();
}
-instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
- uint64_t Weight) {
+void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE,
+ InstrProfValueSiteRecord &Input,
+ uint64_t Weight) {
this->sortByTargetValues();
Input.sortByTargetValues();
auto I = ValueData.begin();
auto IE = ValueData.end();
- instrprof_error Result = instrprof_error::success;
for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
++J) {
while (I != IE && I->Value < J->Value)
@@ -272,92 +348,80 @@ instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
bool Overflowed;
I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
if (Overflowed)
- Result = instrprof_error::counter_overflow;
+ SIPE.addError(instrprof_error::counter_overflow);
++I;
continue;
}
ValueData.insert(I, *J);
}
- return Result;
}
-instrprof_error InstrProfValueSiteRecord::scale(uint64_t Weight) {
- instrprof_error Result = instrprof_error::success;
+void InstrProfValueSiteRecord::scale(SoftInstrProfErrors &SIPE,
+ uint64_t Weight) {
for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
bool Overflowed;
I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
if (Overflowed)
- Result = instrprof_error::counter_overflow;
+ SIPE.addError(instrprof_error::counter_overflow);
}
- return Result;
}
// Merge Value Profile data from Src record to this record for ValueKind.
// Scale merged value counts by \p Weight.
-instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
- InstrProfRecord &Src,
- uint64_t Weight) {
+void InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
+ InstrProfRecord &Src,
+ uint64_t Weight) {
uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
- if (ThisNumValueSites != OtherNumValueSites)
- return instrprof_error::value_site_count_mismatch;
+ if (ThisNumValueSites != OtherNumValueSites) {
+ SIPE.addError(instrprof_error::value_site_count_mismatch);
+ return;
+ }
std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
getValueSitesForKind(ValueKind);
std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
Src.getValueSitesForKind(ValueKind);
- instrprof_error Result = instrprof_error::success;
for (uint32_t I = 0; I < ThisNumValueSites; I++)
- MergeResult(Result, ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight));
- return Result;
+ ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight);
}
-instrprof_error InstrProfRecord::merge(InstrProfRecord &Other,
- uint64_t Weight) {
+void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) {
// If the number of counters doesn't match we either have bad data
// or a hash collision.
- if (Counts.size() != Other.Counts.size())
- return instrprof_error::count_mismatch;
-
- instrprof_error Result = instrprof_error::success;
+ if (Counts.size() != Other.Counts.size()) {
+ SIPE.addError(instrprof_error::count_mismatch);
+ return;
+ }
for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
bool Overflowed;
Counts[I] =
SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
if (Overflowed)
- Result = instrprof_error::counter_overflow;
+ SIPE.addError(instrprof_error::counter_overflow);
}
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
- MergeResult(Result, mergeValueProfData(Kind, Other, Weight));
-
- return Result;
+ mergeValueProfData(Kind, Other, Weight);
}
-instrprof_error InstrProfRecord::scaleValueProfData(uint32_t ValueKind,
- uint64_t Weight) {
+void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) {
uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
getValueSitesForKind(ValueKind);
- instrprof_error Result = instrprof_error::success;
for (uint32_t I = 0; I < ThisNumValueSites; I++)
- MergeResult(Result, ThisSiteRecords[I].scale(Weight));
- return Result;
+ ThisSiteRecords[I].scale(SIPE, Weight);
}
-instrprof_error InstrProfRecord::scale(uint64_t Weight) {
- instrprof_error Result = instrprof_error::success;
+void InstrProfRecord::scale(uint64_t Weight) {
for (auto &Count : this->Counts) {
bool Overflowed;
Count = SaturatingMultiply(Count, Weight, &Overflowed);
- if (Overflowed && Result == instrprof_error::success) {
- Result = instrprof_error::counter_overflow;
- }
+ if (Overflowed)
+ SIPE.addError(instrprof_error::counter_overflow);
}
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
- MergeResult(Result, scaleValueProfData(Kind, Weight));
-
- return Result;
+ scaleValueProfData(Kind, Weight);
}
// Map indirect call target name hash to name string.
@@ -371,8 +435,14 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
std::lower_bound(ValueMap->begin(), ValueMap->end(), Value,
[](const std::pair<uint64_t, uint64_t> &LHS,
uint64_t RHS) { return LHS.first < RHS; });
- if (Result != ValueMap->end())
+ // Raw function pointer collected by value profiler may be from
+ // external functions that are not instrumented. They won't have
+ // mapping data to be used by the deserializer. Force the value to
+ // be 0 in this case.
+ if (Result != ValueMap->end() && Result->first == Value)
Value = (uint64_t)Result->second;
+ else
+ Value = 0;
break;
}
}
@@ -388,7 +458,7 @@ void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
std::vector<InstrProfValueSiteRecord> &ValueSites =
getValueSitesForKind(ValueKind);
if (N == 0)
- ValueSites.push_back(InstrProfValueSiteRecord());
+ ValueSites.emplace_back();
else
ValueSites.emplace_back(VData, VData + N);
}
@@ -422,10 +492,8 @@ uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
}
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
- uint32_t K, uint32_t S,
- uint64_t (*Mapper)(uint32_t, uint64_t)) {
- return reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(
- Dst, K, S, Mapper);
+ uint32_t K, uint32_t S) {
+ reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
}
ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
@@ -436,12 +504,12 @@ ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
}
static ValueProfRecordClosure InstrProfRecordClosure = {
- 0,
+ nullptr,
getNumValueKindsInstrProf,
getNumValueSitesInstrProf,
getNumValueDataInstrProf,
getNumValueDataForSiteInstrProf,
- 0,
+ nullptr,
getValueForSiteInstrProf,
allocValueProfDataInstrProf};
@@ -526,45 +594,45 @@ static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
ValueProfData());
}
-instrprof_error ValueProfData::checkIntegrity() {
+Error ValueProfData::checkIntegrity() {
if (NumValueKinds > IPVK_Last + 1)
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
// Total size needs to be mulltiple of quadword size.
if (TotalSize % sizeof(uint64_t))
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
ValueProfRecord *VR = getFirstValueProfRecord(this);
for (uint32_t K = 0; K < this->NumValueKinds; K++) {
if (VR->Kind > IPVK_Last)
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
VR = getValueProfRecordNext(VR);
if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
}
- return instrprof_error::success;
+ return Error::success();
}
-ErrorOr<std::unique_ptr<ValueProfData>>
+Expected<std::unique_ptr<ValueProfData>>
ValueProfData::getValueProfData(const unsigned char *D,
const unsigned char *const BufferEnd,
support::endianness Endianness) {
using namespace support;
if (D + sizeof(ValueProfData) > BufferEnd)
- return instrprof_error::truncated;
+ return make_error<InstrProfError>(instrprof_error::truncated);
const unsigned char *Header = D;
uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
if (D + TotalSize > BufferEnd)
- return instrprof_error::too_large;
+ return make_error<InstrProfError>(instrprof_error::too_large);
std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
memcpy(VPD.get(), D, TotalSize);
// Byte swap.
VPD->swapBytesToHost(Endianness);
- instrprof_error EC = VPD->checkIntegrity();
- if (EC != instrprof_error::success)
- return EC;
+ Error E = VPD->checkIntegrity();
+ if (E)
+ return std::move(E);
return std::move(VPD);
}
@@ -599,4 +667,117 @@ void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
sys::swapByteOrder<uint32_t>(NumValueKinds);
}
+void annotateValueSite(Module &M, Instruction &Inst,
+ const InstrProfRecord &InstrProfR,
+ InstrProfValueKind ValueKind, uint32_t SiteIdx,
+ uint32_t MaxMDCount) {
+ uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
+ if (!NV)
+ return;
+
+ uint64_t Sum = 0;
+ std::unique_ptr<InstrProfValueData[]> VD =
+ InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
+
+ ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
+ annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
+}
+
+void annotateValueSite(Module &M, Instruction &Inst,
+ ArrayRef<InstrProfValueData> VDs,
+ uint64_t Sum, InstrProfValueKind ValueKind,
+ uint32_t MaxMDCount) {
+ LLVMContext &Ctx = M.getContext();
+ MDBuilder MDHelper(Ctx);
+ SmallVector<Metadata *, 3> Vals;
+ // Tag
+ Vals.push_back(MDHelper.createString("VP"));
+ // Value Kind
+ Vals.push_back(MDHelper.createConstant(
+ ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
+ // Total Count
+ Vals.push_back(
+ MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
+
+ // Value Profile Data
+ uint32_t MDCount = MaxMDCount;
+ for (auto &VD : VDs) {
+ Vals.push_back(MDHelper.createConstant(
+ ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
+ Vals.push_back(MDHelper.createConstant(
+ ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
+ if (--MDCount == 0)
+ break;
+ }
+ Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
+}
+
+bool getValueProfDataFromInst(const Instruction &Inst,
+ InstrProfValueKind ValueKind,
+ uint32_t MaxNumValueData,
+ InstrProfValueData ValueData[],
+ uint32_t &ActualNumValueData, uint64_t &TotalC) {
+ MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
+ if (!MD)
+ return false;
+
+ unsigned NOps = MD->getNumOperands();
+
+ if (NOps < 5)
+ return false;
+
+ // Operand 0 is a string tag "VP":
+ MDString *Tag = cast<MDString>(MD->getOperand(0));
+ if (!Tag)
+ return false;
+
+ if (!Tag->getString().equals("VP"))
+ return false;
+
+ // Now check kind:
+ ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
+ if (!KindInt)
+ return false;
+ if (KindInt->getZExtValue() != ValueKind)
+ return false;
+
+ // Get total count
+ ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
+ if (!TotalCInt)
+ return false;
+ TotalC = TotalCInt->getZExtValue();
+
+ ActualNumValueData = 0;
+
+ for (unsigned I = 3; I < NOps; I += 2) {
+ if (ActualNumValueData >= MaxNumValueData)
+ break;
+ ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
+ ConstantInt *Count =
+ mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
+ if (!Value || !Count)
+ return false;
+ ValueData[ActualNumValueData].Value = Value->getZExtValue();
+ ValueData[ActualNumValueData].Count = Count->getZExtValue();
+ ActualNumValueData++;
+ }
+ return true;
+}
+
+MDNode *getPGOFuncNameMetadata(const Function &F) {
+ return F.getMetadata(getPGOFuncNameMetadataName());
}
+
+void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
+ // Only for internal linkage functions.
+ if (PGOFuncName == F.getName())
+ return;
+ // Don't create duplicated meta-data.
+ if (getPGOFuncNameMetadata(F))
+ return;
+ LLVMContext &C = F.getContext();
+ MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
+ F.setMetadata(getPGOFuncNameMetadataName(), N);
+}
+
+} // end namespace llvm
diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp
index 5e83456822fd..81c13b35ce30 100644
--- a/lib/ProfileData/InstrProfReader.cpp
+++ b/lib/ProfileData/InstrProfReader.cpp
@@ -18,33 +18,33 @@
using namespace llvm;
-static ErrorOr<std::unique_ptr<MemoryBuffer>>
-setupMemoryBuffer(std::string Path) {
+static Expected<std::unique_ptr<MemoryBuffer>>
+setupMemoryBuffer(const Twine &Path) {
ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
MemoryBuffer::getFileOrSTDIN(Path);
if (std::error_code EC = BufferOrErr.getError())
- return EC;
+ return errorCodeToError(EC);
return std::move(BufferOrErr.get());
}
-static std::error_code initializeReader(InstrProfReader &Reader) {
+static Error initializeReader(InstrProfReader &Reader) {
return Reader.readHeader();
}
-ErrorOr<std::unique_ptr<InstrProfReader>>
-InstrProfReader::create(std::string Path) {
+Expected<std::unique_ptr<InstrProfReader>>
+InstrProfReader::create(const Twine &Path) {
// Set up the buffer to read.
auto BufferOrError = setupMemoryBuffer(Path);
- if (std::error_code EC = BufferOrError.getError())
- return EC;
+ if (Error E = BufferOrError.takeError())
+ return std::move(E);
return InstrProfReader::create(std::move(BufferOrError.get()));
}
-ErrorOr<std::unique_ptr<InstrProfReader>>
+Expected<std::unique_ptr<InstrProfReader>>
InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
// Sanity check the buffer.
if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max())
- return instrprof_error::too_large;
+ return make_error<InstrProfError>(instrprof_error::too_large);
std::unique_ptr<InstrProfReader> Result;
// Create the reader.
@@ -57,46 +57,49 @@ InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
else if (TextInstrProfReader::hasFormat(*Buffer))
Result.reset(new TextInstrProfReader(std::move(Buffer)));
else
- return instrprof_error::unrecognized_format;
+ return make_error<InstrProfError>(instrprof_error::unrecognized_format);
// Initialize the reader and return the result.
- if (std::error_code EC = initializeReader(*Result))
- return EC;
+ if (Error E = initializeReader(*Result))
+ return std::move(E);
return std::move(Result);
}
-ErrorOr<std::unique_ptr<IndexedInstrProfReader>>
-IndexedInstrProfReader::create(std::string Path) {
+Expected<std::unique_ptr<IndexedInstrProfReader>>
+IndexedInstrProfReader::create(const Twine &Path) {
// Set up the buffer to read.
auto BufferOrError = setupMemoryBuffer(Path);
- if (std::error_code EC = BufferOrError.getError())
- return EC;
+ if (Error E = BufferOrError.takeError())
+ return std::move(E);
return IndexedInstrProfReader::create(std::move(BufferOrError.get()));
}
-ErrorOr<std::unique_ptr<IndexedInstrProfReader>>
+Expected<std::unique_ptr<IndexedInstrProfReader>>
IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
// Sanity check the buffer.
if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max())
- return instrprof_error::too_large;
+ return make_error<InstrProfError>(instrprof_error::too_large);
// Create the reader.
if (!IndexedInstrProfReader::hasFormat(*Buffer))
- return instrprof_error::bad_magic;
+ return make_error<InstrProfError>(instrprof_error::bad_magic);
auto Result = llvm::make_unique<IndexedInstrProfReader>(std::move(Buffer));
// Initialize the reader and return the result.
- if (std::error_code EC = initializeReader(*Result))
- return EC;
+ if (Error E = initializeReader(*Result))
+ return std::move(E);
return std::move(Result);
}
void InstrProfIterator::Increment() {
- if (Reader->readNextRecord(Record))
+ if (auto E = Reader->readNextRecord(Record)) {
+ // Handle errors in the reader.
+ InstrProfError::take(std::move(E));
*this = InstrProfIterator();
+ }
}
bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) {
@@ -109,12 +112,30 @@ bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) {
[](char c) { return ::isprint(c) || ::isspace(c); });
}
-std::error_code TextInstrProfReader::readHeader() {
+// Read the profile variant flag from the header: ":FE" means this is a FE
+// generated profile. ":IR" means this is an IR level profile. Other strings
+// with a leading ':' will be reported an error format.
+Error TextInstrProfReader::readHeader() {
Symtab.reset(new InstrProfSymtab());
+ bool IsIRInstr = false;
+ if (!Line->startswith(":")) {
+ IsIRLevelProfile = false;
+ return success();
+ }
+ StringRef Str = (Line)->substr(1);
+ if (Str.equals_lower("ir"))
+ IsIRInstr = true;
+ else if (Str.equals_lower("fe"))
+ IsIRInstr = false;
+ else
+ return error(instrprof_error::bad_header);
+
+ ++Line;
+ IsIRLevelProfile = IsIRInstr;
return success();
}
-std::error_code
+Error
TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
#define CHECK_LINE_END(Line) \
@@ -156,7 +177,7 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
std::vector<InstrProfValueData> CurrentValues;
for (uint32_t V = 0; V < NumValueData; V++) {
CHECK_LINE_END(Line);
- std::pair<StringRef, StringRef> VD = Line->split(':');
+ std::pair<StringRef, StringRef> VD = Line->rsplit(':');
uint64_t TakenCount, Value;
if (VK == IPVK_IndirectCallTarget) {
Symtab->addFuncName(VD.first);
@@ -178,7 +199,7 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
#undef VP_READ_ADVANCE
}
-std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) {
+Error TextInstrProfReader::readNextRecord(InstrProfRecord &Record) {
// Skip empty lines and comments.
while (!Line.is_at_end() && (Line->empty() || Line->startswith("#")))
++Line;
@@ -220,8 +241,8 @@ std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) {
}
// Check if value profile data exists and read it if so.
- if (std::error_code EC = readValueProfileData(Record))
- return EC;
+ if (Error E = readValueProfileData(Record))
+ return E;
// This is needed to avoid two pass parsing because llvm-profdata
// does dumping while reading.
@@ -240,7 +261,7 @@ bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) {
}
template <class IntPtrT>
-std::error_code RawInstrProfReader<IntPtrT>::readHeader() {
+Error RawInstrProfReader<IntPtrT>::readHeader() {
if (!hasFormat(*DataBuffer))
return error(instrprof_error::bad_magic);
if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
@@ -252,26 +273,25 @@ std::error_code RawInstrProfReader<IntPtrT>::readHeader() {
}
template <class IntPtrT>
-std::error_code
-RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
+Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
const char *End = DataBuffer->getBufferEnd();
// Skip zero padding between profiles.
while (CurrentPos != End && *CurrentPos == 0)
++CurrentPos;
// If there's nothing left, we're done.
if (CurrentPos == End)
- return instrprof_error::eof;
+ return make_error<InstrProfError>(instrprof_error::eof);
// If there isn't enough space for another header, this is probably just
// garbage at the end of the file.
if (CurrentPos + sizeof(RawInstrProf::Header) > End)
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
// The writer ensures each profile is padded to start at an aligned address.
if (reinterpret_cast<size_t>(CurrentPos) % alignOf<uint64_t>())
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
// The magic should have the same byte order as in the previous header.
uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
- return instrprof_error::bad_magic;
+ return make_error<InstrProfError>(instrprof_error::bad_magic);
// There's another profile to read, so we need to process the header.
auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
@@ -279,30 +299,31 @@ RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
}
template <class IntPtrT>
-void RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
+Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
+ if (Error E = Symtab.create(StringRef(NamesStart, NamesSize)))
+ return error(std::move(E));
for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
- StringRef FunctionName(getName(I->NamePtr), swap(I->NameSize));
- Symtab.addFuncName(FunctionName);
const IntPtrT FPtr = swap(I->FunctionPointer);
if (!FPtr)
continue;
- Symtab.mapAddress(FPtr, IndexedInstrProf::ComputeHash(FunctionName));
+ Symtab.mapAddress(FPtr, I->NameRef);
}
Symtab.finalizeSymtab();
+ return success();
}
template <class IntPtrT>
-std::error_code
-RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) {
- if (swap(Header.Version) != RawInstrProf::Version)
+Error RawInstrProfReader<IntPtrT>::readHeader(
+ const RawInstrProf::Header &Header) {
+ Version = swap(Header.Version);
+ if (GET_VERSION(Version) != RawInstrProf::Version)
return error(instrprof_error::unsupported_version);
CountersDelta = swap(Header.CountersDelta);
NamesDelta = swap(Header.NamesDelta);
auto DataSize = swap(Header.DataSize);
auto CountersSize = swap(Header.CountersSize);
- auto NamesSize = swap(Header.NamesSize);
- auto ValueDataSize = swap(Header.ValueDataSize);
+ NamesSize = swap(Header.NamesSize);
ValueKindLast = swap(Header.ValueKindLast);
auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>);
@@ -312,10 +333,9 @@ RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) {
ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes;
ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize;
ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
- size_t ProfileSize = ValueDataOffset + ValueDataSize;
auto *Start = reinterpret_cast<const char *>(&Header);
- if (Start + ProfileSize > DataBuffer->getBufferEnd())
+ if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
return error(instrprof_error::bad_header);
Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
@@ -324,33 +344,29 @@ RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) {
CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
NamesStart = Start + NamesOffset;
ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
- ProfileEnd = Start + ProfileSize;
std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
- createSymtab(*NewSymtab.get());
+ if (Error E = createSymtab(*NewSymtab.get()))
+ return E;
+
Symtab = std::move(NewSymtab);
return success();
}
template <class IntPtrT>
-std::error_code RawInstrProfReader<IntPtrT>::readName(InstrProfRecord &Record) {
- Record.Name = StringRef(getName(Data->NamePtr), swap(Data->NameSize));
- if (Record.Name.data() < NamesStart ||
- Record.Name.data() + Record.Name.size() >
- reinterpret_cast<const char *>(ValueDataStart))
- return error(instrprof_error::malformed);
+Error RawInstrProfReader<IntPtrT>::readName(InstrProfRecord &Record) {
+ Record.Name = getName(Data->NameRef);
return success();
}
template <class IntPtrT>
-std::error_code RawInstrProfReader<IntPtrT>::readFuncHash(
- InstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readFuncHash(InstrProfRecord &Record) {
Record.Hash = swap(Data->FuncHash);
return success();
}
template <class IntPtrT>
-std::error_code RawInstrProfReader<IntPtrT>::readRawCounts(
+Error RawInstrProfReader<IntPtrT>::readRawCounts(
InstrProfRecord &Record) {
uint32_t NumCounters = swap(Data->NumCounters);
IntPtrT CounterPtr = Data->CounterPtr;
@@ -377,8 +393,8 @@ std::error_code RawInstrProfReader<IntPtrT>::readRawCounts(
}
template <class IntPtrT>
-std::error_code
-RawInstrProfReader<IntPtrT>::readValueProfilingData(InstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readValueProfilingData(
+ InstrProfRecord &Record) {
Record.clearValueData();
CurValueDataSize = 0;
@@ -390,41 +406,44 @@ RawInstrProfReader<IntPtrT>::readValueProfilingData(InstrProfRecord &Record) {
if (!NumValueKinds)
return success();
- ErrorOr<std::unique_ptr<ValueProfData>> VDataPtrOrErr =
- ValueProfData::getValueProfData(ValueDataStart,
- (const unsigned char *)ProfileEnd,
- getDataEndianness());
+ Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr =
+ ValueProfData::getValueProfData(
+ ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(),
+ getDataEndianness());
- if (VDataPtrOrErr.getError())
- return VDataPtrOrErr.getError();
+ if (Error E = VDataPtrOrErr.takeError())
+ return E;
+ // Note that besides deserialization, this also performs the conversion for
+ // indirect call targets. The function pointers from the raw profile are
+ // remapped into function name hashes.
VDataPtrOrErr.get()->deserializeTo(Record, &Symtab->getAddrHashMap());
CurValueDataSize = VDataPtrOrErr.get()->getSize();
return success();
}
template <class IntPtrT>
-std::error_code
-RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) {
+Error RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) {
if (atEnd())
- if (std::error_code EC = readNextHeader(ProfileEnd))
- return EC;
+ // At this point, ValueDataStart field points to the next header.
+ if (Error E = readNextHeader(getNextHeaderPos()))
+ return E;
// Read name ad set it in Record.
- if (std::error_code EC = readName(Record))
- return EC;
+ if (Error E = readName(Record))
+ return E;
// Read FuncHash and set it in Record.
- if (std::error_code EC = readFuncHash(Record))
- return EC;
+ if (Error E = readFuncHash(Record))
+ return E;
// Read raw counts and set Record.
- if (std::error_code EC = readRawCounts(Record))
- return EC;
+ if (Error E = readRawCounts(Record))
+ return E;
// Read value data and set Record.
- if (std::error_code EC = readValueProfilingData(Record))
- return EC;
+ if (Error E = readValueProfilingData(Record))
+ return E;
// Iterate.
advanceData();
@@ -446,10 +465,10 @@ typedef InstrProfLookupTrait::offset_type offset_type;
bool InstrProfLookupTrait::readValueProfilingData(
const unsigned char *&D, const unsigned char *const End) {
- ErrorOr<std::unique_ptr<ValueProfData>> VDataPtrOrErr =
+ Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr =
ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
- if (VDataPtrOrErr.getError())
+ if (VDataPtrOrErr.takeError())
return false;
VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr);
@@ -475,10 +494,10 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
return data_type();
uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
- // Initialize number of counters for FormatVersion == 1.
+ // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
uint64_t CountsSize = N / sizeof(uint64_t) - 1;
// If format version is different then read the number of counters.
- if (FormatVersion != 1) {
+ if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
if (D + sizeof(uint64_t) > End)
return data_type();
CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
@@ -495,7 +514,8 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
// Read value profiling data.
- if (FormatVersion > 2 && !readValueProfilingData(D, End)) {
+ if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
+ !readValueProfilingData(D, End)) {
DataBuffer.clear();
return data_type();
}
@@ -504,31 +524,31 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
}
template <typename HashTableImpl>
-std::error_code InstrProfReaderIndex<HashTableImpl>::getRecords(
+Error InstrProfReaderIndex<HashTableImpl>::getRecords(
StringRef FuncName, ArrayRef<InstrProfRecord> &Data) {
auto Iter = HashTable->find(FuncName);
if (Iter == HashTable->end())
- return instrprof_error::unknown_function;
+ return make_error<InstrProfError>(instrprof_error::unknown_function);
Data = (*Iter);
if (Data.empty())
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
- return instrprof_error::success;
+ return Error::success();
}
template <typename HashTableImpl>
-std::error_code InstrProfReaderIndex<HashTableImpl>::getRecords(
+Error InstrProfReaderIndex<HashTableImpl>::getRecords(
ArrayRef<InstrProfRecord> &Data) {
if (atEnd())
- return instrprof_error::eof;
+ return make_error<InstrProfError>(instrprof_error::eof);
Data = *RecordIterator;
if (Data.empty())
- return instrprof_error::malformed;
+ return make_error<InstrProfError>(instrprof_error::malformed);
- return instrprof_error::success;
+ return Error::success();
}
template <typename HashTableImpl>
@@ -553,7 +573,56 @@ bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
return Magic == IndexedInstrProf::Magic;
}
-std::error_code IndexedInstrProfReader::readHeader() {
+const unsigned char *
+IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
+ const unsigned char *Cur) {
+ using namespace IndexedInstrProf;
+ using namespace support;
+ if (Version >= IndexedInstrProf::Version4) {
+ const IndexedInstrProf::Summary *SummaryInLE =
+ reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
+ uint64_t NFields =
+ endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields);
+ uint64_t NEntries =
+ endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries);
+ uint32_t SummarySize =
+ IndexedInstrProf::Summary::getSize(NFields, NEntries);
+ std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
+ IndexedInstrProf::allocSummary(SummarySize);
+
+ const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
+ uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
+ for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
+ Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]);
+
+ llvm::SummaryEntryVector DetailedSummary;
+ for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
+ const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
+ DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
+ Ent.NumBlocks);
+ }
+ // initialize InstrProfSummary using the SummaryData from disk.
+ this->Summary = llvm::make_unique<ProfileSummary>(
+ ProfileSummary::PSK_Instr, DetailedSummary,
+ SummaryData->get(Summary::TotalBlockCount),
+ SummaryData->get(Summary::MaxBlockCount),
+ SummaryData->get(Summary::MaxInternalBlockCount),
+ SummaryData->get(Summary::MaxFunctionCount),
+ SummaryData->get(Summary::TotalNumBlocks),
+ SummaryData->get(Summary::TotalNumFunctions));
+ return Cur + SummarySize;
+ } else {
+ // For older version of profile data, we need to compute on the fly:
+ using namespace IndexedInstrProf;
+ InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
+ // FIXME: This only computes an empty summary. Need to call addRecord for
+ // all InstrProfRecords to get the correct summary.
+ this->Summary = Builder.getSummary();
+ return Cur;
+ }
+}
+
+Error IndexedInstrProfReader::readHeader() {
const unsigned char *Start =
(const unsigned char *)DataBuffer->getBufferStart();
const unsigned char *Cur = Start;
@@ -572,12 +641,11 @@ std::error_code IndexedInstrProfReader::readHeader() {
// Read the version.
uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version);
- if (FormatVersion > IndexedInstrProf::Version)
+ if (GET_VERSION(FormatVersion) >
+ IndexedInstrProf::ProfVersion::CurrentVersion)
return error(instrprof_error::unsupported_version);
- // Read the maximal function count.
- MaxFunctionCount =
- endian::byte_swap<uint64_t, little>(Header->MaxFunctionCount);
+ Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur);
// Read the hash type and start offset.
IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
@@ -606,13 +674,13 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
return *Symtab.get();
}
-ErrorOr<InstrProfRecord>
+Expected<InstrProfRecord>
IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName,
uint64_t FuncHash) {
ArrayRef<InstrProfRecord> Data;
- std::error_code EC = Index->getRecords(FuncName, Data);
- if (EC != instrprof_error::success)
- return EC;
+ Error Err = Index->getRecords(FuncName, Data);
+ if (Err)
+ return std::move(Err);
// Found it. Look for counters with the right hash.
for (unsigned I = 0, E = Data.size(); I < E; ++I) {
// Check for a match and fill the vector if there is one.
@@ -623,26 +691,25 @@ IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName,
return error(instrprof_error::hash_mismatch);
}
-std::error_code
-IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, uint64_t FuncHash,
- std::vector<uint64_t> &Counts) {
- ErrorOr<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
- if (std::error_code EC = Record.getError())
- return EC;
+Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName,
+ uint64_t FuncHash,
+ std::vector<uint64_t> &Counts) {
+ Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
+ if (Error E = Record.takeError())
+ return error(std::move(E));
Counts = Record.get().Counts;
return success();
}
-std::error_code IndexedInstrProfReader::readNextRecord(
- InstrProfRecord &Record) {
+Error IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) {
static unsigned RecordIndex = 0;
ArrayRef<InstrProfRecord> Data;
- std::error_code EC = Index->getRecords(Data);
- if (EC != instrprof_error::success)
- return error(EC);
+ Error E = Index->getRecords(Data);
+ if (E)
+ return error(std::move(E));
Record = Data[RecordIndex++];
if (RecordIndex >= Data.size()) {
diff --git a/lib/ProfileData/InstrProfWriter.cpp b/lib/ProfileData/InstrProfWriter.cpp
index f5227248af20..e25299ef6706 100644
--- a/lib/ProfileData/InstrProfWriter.cpp
+++ b/lib/ProfileData/InstrProfWriter.cpp
@@ -20,10 +20,59 @@
using namespace llvm;
-namespace {
-static support::endianness ValueProfDataEndianness = support::little;
+// A struct to define how the data stream should be patched. For Indexed
+// profiling, only uint64_t data type is needed.
+struct PatchItem {
+ uint64_t Pos; // Where to patch.
+ uint64_t *D; // Pointer to an array of source data.
+ int N; // Number of elements in \c D array.
+};
+
+namespace llvm {
+// A wrapper class to abstract writer stream with support of bytes
+// back patching.
+class ProfOStream {
-class InstrProfRecordTrait {
+public:
+ ProfOStream(llvm::raw_fd_ostream &FD) : IsFDOStream(true), OS(FD), LE(FD) {}
+ ProfOStream(llvm::raw_string_ostream &STR)
+ : IsFDOStream(false), OS(STR), LE(STR) {}
+
+ uint64_t tell() { return OS.tell(); }
+ void write(uint64_t V) { LE.write<uint64_t>(V); }
+ // \c patch can only be called when all data is written and flushed.
+ // For raw_string_ostream, the patch is done on the target string
+ // directly and it won't be reflected in the stream's internal buffer.
+ void patch(PatchItem *P, int NItems) {
+ using namespace support;
+ if (IsFDOStream) {
+ llvm::raw_fd_ostream &FDOStream = static_cast<llvm::raw_fd_ostream &>(OS);
+ for (int K = 0; K < NItems; K++) {
+ FDOStream.seek(P[K].Pos);
+ for (int I = 0; I < P[K].N; I++)
+ write(P[K].D[I]);
+ }
+ } else {
+ llvm::raw_string_ostream &SOStream =
+ static_cast<llvm::raw_string_ostream &>(OS);
+ std::string &Data = SOStream.str(); // with flush
+ for (int K = 0; K < NItems; K++) {
+ for (int I = 0; I < P[K].N; I++) {
+ uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
+ Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
+ (const char *)&Bytes, sizeof(uint64_t));
+ }
+ }
+ }
+ }
+ // If \c OS is an instance of \c raw_fd_ostream, this field will be
+ // true. Otherwise, \c OS will be an raw_string_ostream.
+ bool IsFDOStream;
+ raw_ostream &OS;
+ support::endian::Writer<support::little> LE;
+};
+
+class InstrProfRecordWriterTrait {
public:
typedef StringRef key_type;
typedef StringRef key_type_ref;
@@ -34,6 +83,10 @@ public:
typedef uint64_t hash_value_type;
typedef uint64_t offset_type;
+ support::endianness ValueProfDataEndianness;
+ InstrProfSummaryBuilder *SummaryBuilder;
+
+ InstrProfRecordWriterTrait() : ValueProfDataEndianness(support::little) {}
static hash_value_type ComputeHash(key_type_ref K) {
return IndexedInstrProf::ComputeHash(K);
}
@@ -61,16 +114,16 @@ public:
return std::make_pair(N, M);
}
- static void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N){
+ void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
Out.write(K.data(), N);
}
- static void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V,
- offset_type) {
+ void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
using namespace llvm::support;
endian::Writer<little> LE(Out);
for (const auto &ProfileData : *V) {
const InstrProfRecord &ProfRecord = ProfileData.second;
+ SummaryBuilder->addRecord(ProfRecord);
LE.write<uint64_t>(ProfileData.first); // Function hash
LE.write<uint64_t>(ProfRecord.Counts.size());
@@ -88,14 +141,22 @@ public:
};
}
+InstrProfWriter::InstrProfWriter(bool Sparse)
+ : Sparse(Sparse), FunctionData(), ProfileKind(PF_Unknown),
+ InfoObj(new InstrProfRecordWriterTrait()) {}
+
+InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
+
// Internal interface for testing purpose only.
void InstrProfWriter::setValueProfDataEndianness(
support::endianness Endianness) {
- ValueProfDataEndianness = Endianness;
+ InfoObj->ValueProfDataEndianness = Endianness;
+}
+void InstrProfWriter::setOutputSparse(bool Sparse) {
+ this->Sparse = Sparse;
}
-std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I,
- uint64_t Weight) {
+Error InstrProfWriter::addRecord(InstrProfRecord &&I, uint64_t Weight) {
auto &ProfileDataMap = FunctionData[I.Name];
bool NewFunc;
@@ -104,73 +165,128 @@ std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I,
ProfileDataMap.insert(std::make_pair(I.Hash, InstrProfRecord()));
InstrProfRecord &Dest = Where->second;
- instrprof_error Result = instrprof_error::success;
if (NewFunc) {
// We've never seen a function with this name and hash, add it.
Dest = std::move(I);
// Fix up the name to avoid dangling reference.
Dest.Name = FunctionData.find(Dest.Name)->getKey();
if (Weight > 1)
- Result = Dest.scale(Weight);
+ Dest.scale(Weight);
} else {
// We're updating a function we've seen before.
- Result = Dest.merge(I, Weight);
+ Dest.merge(I, Weight);
}
Dest.sortValueData();
- // We keep track of the max function count as we go for simplicity.
- // Update this statistic no matter the result of the merge.
- if (Dest.Counts[0] > MaxFunctionCount)
- MaxFunctionCount = Dest.Counts[0];
+ return Dest.takeError();
+}
+
+bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
+ if (!Sparse)
+ return true;
+ for (const auto &Func : PD) {
+ const InstrProfRecord &IPR = Func.second;
+ if (std::any_of(IPR.Counts.begin(), IPR.Counts.end(),
+ [](uint64_t Count) { return Count > 0; }))
+ return true;
+ }
+ return false;
+}
- return Result;
+static void setSummary(IndexedInstrProf::Summary *TheSummary,
+ ProfileSummary &PS) {
+ using namespace IndexedInstrProf;
+ std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
+ TheSummary->NumSummaryFields = Summary::NumKinds;
+ TheSummary->NumCutoffEntries = Res.size();
+ TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
+ TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
+ TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
+ TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
+ TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
+ TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
+ for (unsigned I = 0; I < Res.size(); I++)
+ TheSummary->setEntry(I, Res[I]);
}
-std::pair<uint64_t, uint64_t> InstrProfWriter::writeImpl(raw_ostream &OS) {
- OnDiskChainedHashTableGenerator<InstrProfRecordTrait> Generator;
+void InstrProfWriter::writeImpl(ProfOStream &OS) {
+ OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
+
+ using namespace IndexedInstrProf;
+ InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
+ InfoObj->SummaryBuilder = &ISB;
// Populate the hash table generator.
for (const auto &I : FunctionData)
- Generator.insert(I.getKey(), &I.getValue());
-
- using namespace llvm::support;
- endian::Writer<little> LE(OS);
-
+ if (shouldEncodeData(I.getValue()))
+ Generator.insert(I.getKey(), &I.getValue());
// Write the header.
IndexedInstrProf::Header Header;
Header.Magic = IndexedInstrProf::Magic;
- Header.Version = IndexedInstrProf::Version;
- Header.MaxFunctionCount = MaxFunctionCount;
+ Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
+ if (ProfileKind == PF_IRLevel)
+ Header.Version |= VARIANT_MASK_IR_PROF;
+ Header.Unused = 0;
Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
Header.HashOffset = 0;
int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
- // Only write out all the fields execpt 'HashOffset'. We need
+ // Only write out all the fields except 'HashOffset'. We need
// to remember the offset of that field to allow back patching
// later.
for (int I = 0; I < N - 1; I++)
- LE.write<uint64_t>(reinterpret_cast<uint64_t *>(&Header)[I]);
+ OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
- // Save a space to write the hash table start location.
- uint64_t HashTableStartLoc = OS.tell();
+ // Save the location of Header.HashOffset field in \c OS.
+ uint64_t HashTableStartFieldOffset = OS.tell();
// Reserve the space for HashOffset field.
- LE.write<uint64_t>(0);
+ OS.write(0);
+
+ // Reserve space to write profile summary data.
+ uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
+ uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
+ // Remember the summary offset.
+ uint64_t SummaryOffset = OS.tell();
+ for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
+ OS.write(0);
+
// Write the hash table.
- uint64_t HashTableStart = Generator.Emit(OS);
+ uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
+
+ // Allocate space for data to be serialized out.
+ std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
+ IndexedInstrProf::allocSummary(SummarySize);
+ // Compute the Summary and copy the data to the data
+ // structure to be serialized out (to disk or buffer).
+ std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
+ setSummary(TheSummary.get(), *PS);
+ InfoObj->SummaryBuilder = 0;
- return std::make_pair(HashTableStartLoc, HashTableStart);
+ // Now do the final patch:
+ PatchItem PatchItems[] = {
+ // Patch the Header.HashOffset field.
+ {HashTableStartFieldOffset, &HashTableStart, 1},
+ // Patch the summary data.
+ {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
+ (int)(SummarySize / sizeof(uint64_t))}};
+ OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems));
}
void InstrProfWriter::write(raw_fd_ostream &OS) {
// Write the hash table.
- auto TableStart = writeImpl(OS);
+ ProfOStream POS(OS);
+ writeImpl(POS);
+}
- // Go back and fill in the hash table start.
- using namespace support;
- OS.seek(TableStart.first);
- // Now patch the HashOffset field previously reserved.
- endian::Writer<little>(OS).write<uint64_t>(TableStart.second);
+std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
+ std::string Data;
+ llvm::raw_string_ostream OS(Data);
+ ProfOStream POS(OS);
+ // Write the hash table.
+ writeImpl(POS);
+ // Return this in an aligned memory buffer.
+ return MemoryBuffer::getMemBufferCopy(Data);
}
static const char *ValueProfKindStr[] = {
@@ -218,29 +334,16 @@ void InstrProfWriter::writeRecordInText(const InstrProfRecord &Func,
}
void InstrProfWriter::writeText(raw_fd_ostream &OS) {
+ if (ProfileKind == PF_IRLevel)
+ OS << "# IR level Instrumentation Flag\n:ir\n";
InstrProfSymtab Symtab;
for (const auto &I : FunctionData)
- Symtab.addFuncName(I.getKey());
+ if (shouldEncodeData(I.getValue()))
+ Symtab.addFuncName(I.getKey());
Symtab.finalizeSymtab();
for (const auto &I : FunctionData)
- for (const auto &Func : I.getValue())
- writeRecordInText(Func.second, Symtab, OS);
-}
-
-std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
- std::string Data;
- llvm::raw_string_ostream OS(Data);
- // Write the hash table.
- auto TableStart = writeImpl(OS);
- OS.flush();
-
- // Go back and fill in the hash table start.
- using namespace support;
- uint64_t Bytes = endian::byte_swap<uint64_t, little>(TableStart.second);
- Data.replace(TableStart.first, sizeof(uint64_t), (const char *)&Bytes,
- sizeof(uint64_t));
-
- // Return this in an aligned memory buffer.
- return MemoryBuffer::getMemBufferCopy(Data);
+ if (shouldEncodeData(I.getValue()))
+ for (const auto &Func : I.getValue())
+ writeRecordInText(Func.second, Symtab, OS);
}
diff --git a/lib/ProfileData/LLVMBuild.txt b/lib/ProfileData/LLVMBuild.txt
index a7f471fc582e..b3d749feda61 100644
--- a/lib/ProfileData/LLVMBuild.txt
+++ b/lib/ProfileData/LLVMBuild.txt
@@ -15,8 +15,11 @@
;
;===------------------------------------------------------------------------===;
+[common]
+subdirectories = Coverage
+
[component_0]
type = Library
name = ProfileData
parent = Libraries
-required_libraries = Core Support Object
+required_libraries = Core Support
diff --git a/lib/ProfileData/Makefile b/lib/ProfileData/Makefile
deleted file mode 100644
index 26743612d3d7..000000000000
--- a/lib/ProfileData/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-##===- lib/ProfileData/Makefile ----------------------------*- Makefile -*-===##
-#
-# The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../..
-LIBRARYNAME = LLVMProfileData
-BUILD_ARCHIVE := 1
-
-include $(LEVEL)/Makefile.common
diff --git a/lib/ProfileData/ProfileSummaryBuilder.cpp b/lib/ProfileData/ProfileSummaryBuilder.cpp
new file mode 100644
index 000000000000..f8c3717007b3
--- /dev/null
+++ b/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -0,0 +1,116 @@
+//=-- ProfilesummaryBuilder.cpp - Profile summary computation ---------------=//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for computing profile summary data.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Type.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/ProfileCommon.h"
+#include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Casting.h"
+
+using namespace llvm;
+
+// A set of cutoff values. Each value, when divided by ProfileSummary::Scale
+// (which is 1000000) is a desired percentile of total counts.
+static const uint32_t DefaultCutoffsData[] = {
+ 10000, /* 1% */
+ 100000, /* 10% */
+ 200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000,
+ 800000, 900000, 950000, 990000, 999000, 999900, 999990, 999999};
+const ArrayRef<uint32_t> ProfileSummaryBuilder::DefaultCutoffs =
+ DefaultCutoffsData;
+
+void InstrProfSummaryBuilder::addRecord(const InstrProfRecord &R) {
+ // The first counter is not necessarily an entry count for IR
+ // instrumentation profiles.
+ // Eventually MaxFunctionCount will become obsolete and this can be
+ // removed.
+ addEntryCount(R.Counts[0]);
+ for (size_t I = 1, E = R.Counts.size(); I < E; ++I)
+ addInternalCount(R.Counts[I]);
+}
+
+// To compute the detailed summary, we consider each line containing samples as
+// equivalent to a block with a count in the instrumented profile.
+void SampleProfileSummaryBuilder::addRecord(
+ const sampleprof::FunctionSamples &FS) {
+ NumFunctions++;
+ if (FS.getHeadSamples() > MaxFunctionCount)
+ MaxFunctionCount = FS.getHeadSamples();
+ for (const auto &I : FS.getBodySamples())
+ addCount(I.second.getSamples());
+}
+
+// The argument to this method is a vector of cutoff percentages and the return
+// value is a vector of (Cutoff, MinCount, NumCounts) triplets.
+void ProfileSummaryBuilder::computeDetailedSummary() {
+ if (DetailedSummaryCutoffs.empty())
+ return;
+ auto Iter = CountFrequencies.begin();
+ auto End = CountFrequencies.end();
+ std::sort(DetailedSummaryCutoffs.begin(), DetailedSummaryCutoffs.end());
+
+ uint32_t CountsSeen = 0;
+ uint64_t CurrSum = 0, Count = 0;
+
+ for (uint32_t Cutoff : DetailedSummaryCutoffs) {
+ assert(Cutoff <= 999999);
+ APInt Temp(128, TotalCount);
+ APInt N(128, Cutoff);
+ APInt D(128, ProfileSummary::Scale);
+ Temp *= N;
+ Temp = Temp.sdiv(D);
+ uint64_t DesiredCount = Temp.getZExtValue();
+ assert(DesiredCount <= TotalCount);
+ while (CurrSum < DesiredCount && Iter != End) {
+ Count = Iter->first;
+ uint32_t Freq = Iter->second;
+ CurrSum += (Count * Freq);
+ CountsSeen += Freq;
+ Iter++;
+ }
+ assert(CurrSum >= DesiredCount);
+ ProfileSummaryEntry PSE = {Cutoff, Count, CountsSeen};
+ DetailedSummary.push_back(PSE);
+ }
+}
+
+std::unique_ptr<ProfileSummary> SampleProfileSummaryBuilder::getSummary() {
+ computeDetailedSummary();
+ return llvm::make_unique<ProfileSummary>(
+ ProfileSummary::PSK_Sample, DetailedSummary, TotalCount, MaxCount, 0,
+ MaxFunctionCount, NumCounts, NumFunctions);
+}
+
+std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() {
+ computeDetailedSummary();
+ return llvm::make_unique<ProfileSummary>(
+ ProfileSummary::PSK_Instr, DetailedSummary, TotalCount, MaxCount,
+ MaxInternalBlockCount, MaxFunctionCount, NumCounts, NumFunctions);
+}
+
+void InstrProfSummaryBuilder::addEntryCount(uint64_t Count) {
+ addCount(Count);
+ NumFunctions++;
+ if (Count > MaxFunctionCount)
+ MaxFunctionCount = Count;
+}
+
+void InstrProfSummaryBuilder::addInternalCount(uint64_t Count) {
+ addCount(Count);
+ if (Count > MaxInternalBlockCount)
+ MaxInternalBlockCount = Count;
+}
diff --git a/lib/ProfileData/SampleProf.cpp b/lib/ProfileData/SampleProf.cpp
index 9ded757f2b28..cb0246113d80 100644
--- a/lib/ProfileData/SampleProf.cpp
+++ b/lib/ProfileData/SampleProf.cpp
@@ -20,6 +20,9 @@ using namespace llvm::sampleprof;
using namespace llvm;
namespace {
+// FIXME: This class is only here to support the transition to llvm::Error. It
+// will be removed once this transition is complete. Clients should prefer to
+// deal with the Error value directly, rather than converting to error_code.
class SampleProfErrorCategoryType : public std::error_category {
const char *name() const LLVM_NOEXCEPT override { return "llvm.sampleprof"; }
std::string message(int IE) const override {
@@ -71,20 +74,7 @@ raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
return OS;
}
-void LineLocation::dump() const { print(dbgs()); }
-
-void CallsiteLocation::print(raw_ostream &OS) const {
- LineLocation::print(OS);
- OS << ": inlined callee: " << CalleeName;
-}
-
-void CallsiteLocation::dump() const { print(dbgs()); }
-
-inline raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
- const CallsiteLocation &Loc) {
- Loc.print(OS);
- return OS;
-}
+LLVM_DUMP_METHOD void LineLocation::dump() const { print(dbgs()); }
/// \brief Print the sample record to the stream \p OS indented by \p Indent.
void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
@@ -97,7 +87,7 @@ void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
OS << "\n";
}
-void SampleRecord::dump() const { print(dbgs(), 0); }
+LLVM_DUMP_METHOD void SampleRecord::dump() const { print(dbgs(), 0); }
raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
const SampleRecord &Sample) {
@@ -127,11 +117,11 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
OS.indent(Indent);
if (CallsiteSamples.size() > 0) {
OS << "Samples collected in inlined callsites {\n";
- SampleSorter<CallsiteLocation, FunctionSamples> SortedCallsiteSamples(
+ SampleSorter<LineLocation, FunctionSamples> SortedCallsiteSamples(
CallsiteSamples);
for (const auto &CS : SortedCallsiteSamples.get()) {
OS.indent(Indent + 2);
- OS << CS->first << ": ";
+ OS << CS->first << ": inlined callee: " << CS->second.getName() << ": ";
CS->second.print(OS, Indent + 4);
}
OS << "}\n";
diff --git a/lib/ProfileData/SampleProfReader.cpp b/lib/ProfileData/SampleProfReader.cpp
index 93cd87bb82f8..af80b036a5bb 100644
--- a/lib/ProfileData/SampleProfReader.cpp
+++ b/lib/ProfileData/SampleProfReader.cpp
@@ -22,7 +22,7 @@
#include "llvm/ProfileData/SampleProfReader.h"
#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/LEB128.h"
@@ -68,11 +68,8 @@ static bool ParseHead(const StringRef &Input, StringRef &FName,
return true;
}
-
/// \brief Returns true if line offset \p L is legal (only has 16 bits).
-static bool isOffsetLegal(unsigned L) {
- return (L & 0xffff) == L;
-}
+static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
/// \brief Parse \p Input as line sample.
///
@@ -180,6 +177,7 @@ std::error_code SampleProfileReaderText::read() {
}
Profiles[FName] = FunctionSamples();
FunctionSamples &FProfile = Profiles[FName];
+ FProfile.setName(FName);
MergeResult(Result, FProfile.addTotalSamples(NumSamples));
MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
InlineStack.clear();
@@ -202,7 +200,8 @@ std::error_code SampleProfileReaderText::read() {
InlineStack.pop_back();
}
FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
- CallsiteLocation(LineOffset, Discriminator, FName));
+ LineLocation(LineOffset, Discriminator));
+ FSamples.setName(FName);
MergeResult(Result, FSamples.addTotalSamples(NumSamples));
InlineStack.push_back(&FSamples);
} else {
@@ -220,6 +219,8 @@ std::error_code SampleProfileReaderText::read() {
}
}
}
+ if (Result == sampleprof_error::success)
+ computeSummary();
return Result;
}
@@ -351,8 +352,9 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
if (std::error_code EC = FName.getError())
return EC;
- FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
- CallsiteLocation(*LineOffset, *Discriminator, *FName));
+ FunctionSamples &CalleeProfile =
+ FProfile.functionSamplesAt(LineLocation(*LineOffset, *Discriminator));
+ CalleeProfile.setName(*FName);
if (std::error_code EC = readProfile(CalleeProfile))
return EC;
}
@@ -372,6 +374,7 @@ std::error_code SampleProfileReaderBinary::read() {
Profiles[*FName] = FunctionSamples();
FunctionSamples &FProfile = Profiles[*FName];
+ FProfile.setName(*FName);
FProfile.addHeadSamples(*NumHeadSamples);
@@ -400,6 +403,9 @@ std::error_code SampleProfileReaderBinary::readHeader() {
else if (*Version != SPVersion())
return sampleprof_error::unsupported_version;
+ if (std::error_code EC = readSummary())
+ return EC;
+
// Read the name table.
auto Size = readNumber<uint32_t>();
if (std::error_code EC = Size.getError())
@@ -415,6 +421,62 @@ std::error_code SampleProfileReaderBinary::readHeader() {
return sampleprof_error::success;
}
+std::error_code SampleProfileReaderBinary::readSummaryEntry(
+ std::vector<ProfileSummaryEntry> &Entries) {
+ auto Cutoff = readNumber<uint64_t>();
+ if (std::error_code EC = Cutoff.getError())
+ return EC;
+
+ auto MinBlockCount = readNumber<uint64_t>();
+ if (std::error_code EC = MinBlockCount.getError())
+ return EC;
+
+ auto NumBlocks = readNumber<uint64_t>();
+ if (std::error_code EC = NumBlocks.getError())
+ return EC;
+
+ Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderBinary::readSummary() {
+ auto TotalCount = readNumber<uint64_t>();
+ if (std::error_code EC = TotalCount.getError())
+ return EC;
+
+ auto MaxBlockCount = readNumber<uint64_t>();
+ if (std::error_code EC = MaxBlockCount.getError())
+ return EC;
+
+ auto MaxFunctionCount = readNumber<uint64_t>();
+ if (std::error_code EC = MaxFunctionCount.getError())
+ return EC;
+
+ auto NumBlocks = readNumber<uint64_t>();
+ if (std::error_code EC = NumBlocks.getError())
+ return EC;
+
+ auto NumFunctions = readNumber<uint64_t>();
+ if (std::error_code EC = NumFunctions.getError())
+ return EC;
+
+ auto NumSummaryEntries = readNumber<uint64_t>();
+ if (std::error_code EC = NumSummaryEntries.getError())
+ return EC;
+
+ std::vector<ProfileSummaryEntry> Entries;
+ for (unsigned i = 0; i < *NumSummaryEntries; i++) {
+ std::error_code EC = readSummaryEntry(Entries);
+ if (EC != sampleprof_error::success)
+ return EC;
+ }
+ Summary = llvm::make_unique<ProfileSummary>(
+ ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,
+ *MaxFunctionCount, *NumBlocks, *NumFunctions);
+
+ return sampleprof_error::success;
+}
+
bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) {
const uint8_t *Data =
reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
@@ -518,6 +580,7 @@ std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))
return EC;
+ computeSummary();
return sampleprof_error::success;
}
@@ -562,8 +625,9 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
uint32_t LineOffset = Offset >> 16;
uint32_t Discriminator = Offset & 0xffff;
FProfile = &CallerProfile->functionSamplesAt(
- CallsiteLocation(LineOffset, Discriminator, Name));
+ LineLocation(LineOffset, Discriminator));
}
+ FProfile->setName(Name);
for (uint32_t I = 0; I < NumPosCounts; ++I) {
uint32_t Offset;
@@ -669,7 +733,7 @@ bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
///
/// \returns an error code indicating the status of the buffer.
static ErrorOr<std::unique_ptr<MemoryBuffer>>
-setupMemoryBuffer(std::string Filename) {
+setupMemoryBuffer(const Twine &Filename) {
auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = BufferOrErr.getError())
return EC;
@@ -692,7 +756,7 @@ setupMemoryBuffer(std::string Filename) {
///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReader>>
-SampleProfileReader::create(StringRef Filename, LLVMContext &C) {
+SampleProfileReader::create(const Twine &Filename, LLVMContext &C) {
auto BufferOrError = setupMemoryBuffer(Filename);
if (std::error_code EC = BufferOrError.getError())
return EC;
@@ -725,3 +789,14 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
return std::move(Reader);
}
+
+// For text and GCC file formats, we compute the summary after reading the
+// profile. Binary format has the profile summary in its header.
+void SampleProfileReader::computeSummary() {
+ SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
+ for (const auto &I : Profiles) {
+ const FunctionSamples &Profile = I.second;
+ Builder.addRecord(Profile);
+ }
+ Summary = Builder.getSummary();
+}
diff --git a/lib/ProfileData/SampleProfWriter.cpp b/lib/ProfileData/SampleProfWriter.cpp
index 51feee5ad7d1..4fa71288f8d9 100644
--- a/lib/ProfileData/SampleProfWriter.cpp
+++ b/lib/ProfileData/SampleProfWriter.cpp
@@ -37,11 +37,9 @@ using namespace llvm;
///
/// The format used here is more structured and deliberate because
/// it needs to be parsed by the SampleProfileReaderText class.
-std::error_code SampleProfileWriterText::write(StringRef FName,
- const FunctionSamples &S) {
+std::error_code SampleProfileWriterText::write(const FunctionSamples &S) {
auto &OS = *OutputStream;
-
- OS << FName << ":" << S.getTotalSamples();
+ OS << S.getName() << ":" << S.getTotalSamples();
if (Indent == 0)
OS << ":" << S.getHeadSamples();
OS << "\n";
@@ -63,18 +61,18 @@ std::error_code SampleProfileWriterText::write(StringRef FName,
OS << "\n";
}
- SampleSorter<CallsiteLocation, FunctionSamples> SortedCallsiteSamples(
+ SampleSorter<LineLocation, FunctionSamples> SortedCallsiteSamples(
S.getCallsiteSamples());
Indent += 1;
for (const auto &I : SortedCallsiteSamples.get()) {
- CallsiteLocation Loc = I->first;
+ LineLocation Loc = I->first;
const FunctionSamples &CalleeSamples = I->second;
OS.indent(Indent);
if (Loc.Discriminator == 0)
OS << Loc.LineOffset << ": ";
else
OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
- if (std::error_code EC = write(Loc.CalleeName, CalleeSamples))
+ if (std::error_code EC = write(CalleeSamples))
return EC;
}
Indent -= 1;
@@ -105,9 +103,8 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
// Recursively add all the names for inlined callsites.
for (const auto &J : S.getCallsiteSamples()) {
- CallsiteLocation Loc = J.first;
const FunctionSamples &CalleeSamples = J.second;
- addName(Loc.CalleeName);
+ addName(CalleeSamples.getName());
addNames(CalleeSamples);
}
}
@@ -120,6 +117,10 @@ std::error_code SampleProfileWriterBinary::writeHeader(
encodeULEB128(SPMagic(), OS);
encodeULEB128(SPVersion(), OS);
+ computeSummary(ProfileMap);
+ if (auto EC = writeSummary())
+ return EC;
+
// Generate the name table for all the functions referenced in the profile.
for (const auto &I : ProfileMap) {
addName(I.first());
@@ -132,15 +133,29 @@ std::error_code SampleProfileWriterBinary::writeHeader(
OS << N.first;
encodeULEB128(0, OS);
}
-
return sampleprof_error::success;
}
-std::error_code SampleProfileWriterBinary::writeBody(StringRef FName,
- const FunctionSamples &S) {
+std::error_code SampleProfileWriterBinary::writeSummary() {
+ auto &OS = *OutputStream;
+ encodeULEB128(Summary->getTotalCount(), OS);
+ encodeULEB128(Summary->getMaxCount(), OS);
+ encodeULEB128(Summary->getMaxFunctionCount(), OS);
+ encodeULEB128(Summary->getNumCounts(), OS);
+ encodeULEB128(Summary->getNumFunctions(), OS);
+ std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary();
+ encodeULEB128(Entries.size(), OS);
+ for (auto Entry : Entries) {
+ encodeULEB128(Entry.Cutoff, OS);
+ encodeULEB128(Entry.MinCount, OS);
+ encodeULEB128(Entry.NumCounts, OS);
+ }
+ return sampleprof_error::success;
+}
+std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
auto &OS = *OutputStream;
- if (std::error_code EC = writeNameIdx(FName))
+ if (std::error_code EC = writeNameIdx(S.getName()))
return EC;
encodeULEB128(S.getTotalSamples(), OS);
@@ -166,11 +181,11 @@ std::error_code SampleProfileWriterBinary::writeBody(StringRef FName,
// Recursively emit all the callsite samples.
encodeULEB128(S.getCallsiteSamples().size(), OS);
for (const auto &J : S.getCallsiteSamples()) {
- CallsiteLocation Loc = J.first;
+ LineLocation Loc = J.first;
const FunctionSamples &CalleeSamples = J.second;
encodeULEB128(Loc.LineOffset, OS);
encodeULEB128(Loc.Discriminator, OS);
- if (std::error_code EC = writeBody(Loc.CalleeName, CalleeSamples))
+ if (std::error_code EC = writeBody(CalleeSamples))
return EC;
}
@@ -180,10 +195,9 @@ std::error_code SampleProfileWriterBinary::writeBody(StringRef FName,
/// \brief Write samples of a top-level function to a binary file.
///
/// \returns true if the samples were written successfully, false otherwise.
-std::error_code SampleProfileWriterBinary::write(StringRef FName,
- const FunctionSamples &S) {
+std::error_code SampleProfileWriterBinary::write(const FunctionSamples &S) {
encodeULEB128(S.getHeadSamples(), *OutputStream);
- return writeBody(FName, S);
+ return writeBody(S);
}
/// \brief Create a sample profile file writer based on the specified format.
@@ -238,3 +252,13 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
return std::move(Writer);
}
+
+void SampleProfileWriter::computeSummary(
+ const StringMap<FunctionSamples> &ProfileMap) {
+ SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
+ for (const auto &I : ProfileMap) {
+ const FunctionSamples &Profile = I.second;
+ Builder.addRecord(Profile);
+ }
+ Summary = Builder.getSummary();
+}
generated by cgit v1.3 (git 2.53.0) at 2026-04-21 20:16:45 +0000