aboutsummaryrefslogtreecommitdiff
path: root/lib/ProfileData
diff options
context:
space:
mode:
Diffstat (limited to 'lib/ProfileData')
-rw-r--r--lib/ProfileData/Coverage/CoverageMapping.cpp60
-rw-r--r--lib/ProfileData/Coverage/CoverageMappingReader.cpp20
-rw-r--r--lib/ProfileData/Coverage/CoverageMappingWriter.cpp10
-rw-r--r--lib/ProfileData/GCOV.cpp12
-rw-r--r--lib/ProfileData/InstrProf.cpp18
-rw-r--r--lib/ProfileData/InstrProfReader.cpp44
-rw-r--r--lib/ProfileData/InstrProfWriter.cpp2
-rw-r--r--lib/ProfileData/ProfileSummaryBuilder.cpp4
-rw-r--r--lib/ProfileData/SampleProf.cpp56
-rw-r--r--lib/ProfileData/SampleProfReader.cpp447
-rw-r--r--lib/ProfileData/SampleProfWriter.cpp279
11 files changed, 813 insertions, 139 deletions
diff --git a/lib/ProfileData/Coverage/CoverageMapping.cpp b/lib/ProfileData/Coverage/CoverageMapping.cpp
index afd6618e7cb3..8d5e56e26c0f 100644
--- a/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -194,6 +194,15 @@ void FunctionRecordIterator::skipOtherFiles() {
*this = FunctionRecordIterator();
}
+ArrayRef<unsigned> CoverageMapping::getImpreciseRecordIndicesForFilename(
+ StringRef Filename) const {
+ size_t FilenameHash = hash_value(Filename);
+ auto RecordIt = FilenameHash2RecordIndices.find(FilenameHash);
+ if (RecordIt == FilenameHash2RecordIndices.end())
+ return {};
+ return RecordIt->second;
+}
+
Error CoverageMapping::loadFunctionRecord(
const CoverageMappingRecord &Record,
IndexedInstrProfReader &ProfileReader) {
@@ -249,6 +258,20 @@ Error CoverageMapping::loadFunctionRecord(
return Error::success();
Functions.push_back(std::move(Function));
+
+ // Performance optimization: keep track of the indices of the function records
+ // which correspond to each filename. This can be used to substantially speed
+ // up queries for coverage info in a file.
+ unsigned RecordIndex = Functions.size() - 1;
+ for (StringRef Filename : Record.Filenames) {
+ auto &RecordIndices = FilenameHash2RecordIndices[hash_value(Filename)];
+ // Note that there may be duplicates in the filename set for a function
+ // record, because of e.g. macro expansions in the function in which both
+ // the macro and the function are defined in the same file.
+ if (RecordIndices.empty() || RecordIndices.back() != RecordIndex)
+ RecordIndices.push_back(RecordIndex);
+ }
+
return Error::success();
}
@@ -270,6 +293,16 @@ Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
return std::move(Coverage);
}
+// If E is a no_data_found error, returns success. Otherwise returns E.
+static Error handleMaybeNoDataFoundError(Error E) {
+ return handleErrors(
+ std::move(E), [](const CoverageMapError &CME) {
+ if (CME.get() == coveragemap_error::no_data_found)
+ return static_cast<Error>(Error::success());
+ return make_error<CoverageMapError>(CME.get());
+ });
+}
+
Expected<std::unique_ptr<CoverageMapping>>
CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames,
StringRef ProfileFilename, ArrayRef<StringRef> Arches) {
@@ -289,12 +322,21 @@ CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames,
CovMappingBufOrErr.get()->getMemBufferRef();
auto CoverageReadersOrErr =
BinaryCoverageReader::create(CovMappingBufRef, Arch, Buffers);
- if (Error E = CoverageReadersOrErr.takeError())
- return std::move(E);
+ if (Error E = CoverageReadersOrErr.takeError()) {
+ E = handleMaybeNoDataFoundError(std::move(E));
+ if (E)
+ return std::move(E);
+ // E == success (originally a no_data_found error).
+ continue;
+ }
for (auto &Reader : CoverageReadersOrErr.get())
Readers.push_back(std::move(Reader));
Buffers.push_back(std::move(CovMappingBufOrErr.get()));
}
+ // If no readers were created, either no objects were provided or none of them
+ // had coverage data. Return an error in the latter case.
+ if (Readers.empty() && !ObjectFilenames.empty())
+ return make_error<CoverageMapError>(coveragemap_error::no_data_found);
return load(Readers, *ProfileReader);
}
@@ -607,7 +649,12 @@ CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const {
CoverageData FileCoverage(Filename);
std::vector<CountedRegion> Regions;
- for (const auto &Function : Functions) {
+ // Look up the function records in the given file. Due to hash collisions on
+ // the filename, we may get back some records that are not in the file.
+ ArrayRef<unsigned> RecordIndices =
+ getImpreciseRecordIndicesForFilename(Filename);
+ for (unsigned RecordIndex : RecordIndices) {
+ const FunctionRecord &Function = Functions[RecordIndex];
auto MainFileID = findMainViewFileID(Filename, Function);
auto FileIDs = gatherFileIDs(Filename, Function);
for (const auto &CR : Function.CountedRegions)
@@ -627,7 +674,12 @@ CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const {
std::vector<InstantiationGroup>
CoverageMapping::getInstantiationGroups(StringRef Filename) const {
FunctionInstantiationSetCollector InstantiationSetCollector;
- for (const auto &Function : Functions) {
+ // Look up the function records in the given file. Due to hash collisions on
+ // the filename, we may get back some records that are not in the file.
+ ArrayRef<unsigned> RecordIndices =
+ getImpreciseRecordIndicesForFilename(Filename);
+ for (unsigned RecordIndex : RecordIndices) {
+ const FunctionRecord &Function = Functions[RecordIndex];
auto MainFileID = findMainViewFileID(Filename, Function);
if (!MainFileID)
continue;
diff --git a/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index e193e10f91d9..679ff3525eeb 100644
--- a/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -506,7 +506,7 @@ public:
return make_error<CoverageMapError>(coveragemap_error::malformed);
// Each coverage map has an alignment of 8, so we need to adjust alignment
// before reading the next map.
- Buf += alignmentAdjustment(Buf, 8);
+ Buf += offsetToAlignedAddr(Buf, Align(8));
auto CFR = reinterpret_cast<const FuncRecordType *>(FunBuf);
while ((const char *)CFR < FunEnd) {
@@ -539,7 +539,7 @@ Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get(
switch (Version) {
case CovMapVersion::Version1:
- return llvm::make_unique<VersionedCovMapFuncRecordReader<
+ return std::make_unique<VersionedCovMapFuncRecordReader<
CovMapVersion::Version1, IntPtrT, Endian>>(P, R, F);
case CovMapVersion::Version2:
case CovMapVersion::Version3:
@@ -547,10 +547,10 @@ Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get(
if (Error E = P.create(P.getNameData()))
return std::move(E);
if (Version == CovMapVersion::Version2)
- return llvm::make_unique<VersionedCovMapFuncRecordReader<
+ return std::make_unique<VersionedCovMapFuncRecordReader<
CovMapVersion::Version2, IntPtrT, Endian>>(P, R, F);
else
- return llvm::make_unique<VersionedCovMapFuncRecordReader<
+ return std::make_unique<VersionedCovMapFuncRecordReader<
CovMapVersion::Version3, IntPtrT, Endian>>(P, R, F);
}
llvm_unreachable("Unsupported version");
@@ -648,7 +648,7 @@ loadTestingFormat(StringRef Data) {
// Skip the padding bytes because coverage map data has an alignment of 8.
if (CoverageMapping.empty())
return make_error<CoverageMapError>(coveragemap_error::truncated);
- size_t Pad = alignmentAdjustment(CoverageMapping.data(), 8);
+ size_t Pad = offsetToAlignedAddr(CoverageMapping.data(), Align(8));
if (CoverageMapping.size() < Pad)
return make_error<CoverageMapError>(coveragemap_error::malformed);
CoverageMapping = CoverageMapping.substr(Pad);
@@ -666,11 +666,11 @@ static Expected<SectionRef> lookupSection(ObjectFile &OF, StringRef Name) {
};
Name = stripSuffix(Name);
- StringRef FoundName;
for (const auto &Section : OF.sections()) {
- if (auto EC = Section.getName(FoundName))
- return errorCodeToError(EC);
- if (stripSuffix(FoundName) == Name)
+ Expected<StringRef> NameOrErr = Section.getName();
+ if (!NameOrErr)
+ return NameOrErr.takeError();
+ if (stripSuffix(*NameOrErr) == Name)
return Section;
}
return make_error<CoverageMapError>(coveragemap_error::no_data_found);
@@ -682,7 +682,7 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch) {
if (auto *Universal = dyn_cast<MachOUniversalBinary>(Bin.get())) {
// If we have a universal binary, try to look up the object for the
// appropriate architecture.
- auto ObjectFileOrErr = Universal->getObjectForArch(Arch);
+ auto ObjectFileOrErr = Universal->getMachOObjectForArch(Arch);
if (!ObjectFileOrErr)
return ObjectFileOrErr.takeError();
OF = std::move(ObjectFileOrErr.get());
diff --git a/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index 432b20f217ca..d75854a60d1e 100644
--- a/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -24,6 +24,16 @@
using namespace llvm;
using namespace coverage;
+CoverageFilenamesSectionWriter::CoverageFilenamesSectionWriter(
+ ArrayRef<StringRef> Filenames)
+ : Filenames(Filenames) {
+#ifndef NDEBUG
+ StringSet<> NameSet;
+ for (StringRef Name : Filenames)
+ assert(NameSet.insert(Name).second && "Duplicate filename");
+#endif
+}
+
void CoverageFilenamesSectionWriter::write(raw_ostream &OS) {
encodeULEB128(Filenames.size(), OS);
for (const auto &Filename : Filenames) {
diff --git a/lib/ProfileData/GCOV.cpp b/lib/ProfileData/GCOV.cpp
index fa4e433d7aa6..00e6294c57a6 100644
--- a/lib/ProfileData/GCOV.cpp
+++ b/lib/ProfileData/GCOV.cpp
@@ -40,7 +40,7 @@ bool GCOVFile::readGCNO(GCOVBuffer &Buffer) {
while (true) {
if (!Buffer.readFunctionTag())
break;
- auto GFun = make_unique<GCOVFunction>(*this);
+ auto GFun = std::make_unique<GCOVFunction>(*this);
if (!GFun->readGCNO(Buffer, Version))
return false;
Functions.push_back(std::move(GFun));
@@ -164,7 +164,7 @@ bool GCOVFunction::readGCNO(GCOVBuffer &Buff, GCOV::GCOVVersion Version) {
for (uint32_t i = 0, e = BlockCount; i != e; ++i) {
if (!Buff.readInt(Dummy))
return false; // Block flags;
- Blocks.push_back(make_unique<GCOVBlock>(*this, i));
+ Blocks.push_back(std::make_unique<GCOVBlock>(*this, i));
}
// read edges.
@@ -185,7 +185,7 @@ bool GCOVFunction::readGCNO(GCOVBuffer &Buff, GCOV::GCOVVersion Version) {
uint32_t Dst;
if (!Buff.readInt(Dst))
return false;
- Edges.push_back(make_unique<GCOVEdge>(*Blocks[BlockNo], *Blocks[Dst]));
+ Edges.push_back(std::make_unique<GCOVEdge>(*Blocks[BlockNo], *Blocks[Dst]));
GCOVEdge *Edge = Edges.back().get();
Blocks[BlockNo]->addDstEdge(Edge);
Blocks[Dst]->addSrcEdge(Edge);
@@ -702,14 +702,14 @@ std::string FileInfo::getCoveragePath(StringRef Filename,
std::unique_ptr<raw_ostream>
FileInfo::openCoveragePath(StringRef CoveragePath) {
if (Options.NoOutput)
- return llvm::make_unique<raw_null_ostream>();
+ return std::make_unique<raw_null_ostream>();
std::error_code EC;
auto OS =
- llvm::make_unique<raw_fd_ostream>(CoveragePath, EC, sys::fs::F_Text);
+ std::make_unique<raw_fd_ostream>(CoveragePath, EC, sys::fs::OF_Text);
if (EC) {
errs() << EC.message() << "\n";
- return llvm::make_unique<raw_null_ostream>();
+ return std::make_unique<raw_null_ostream>();
}
return std::move(OS);
}
diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp
index 510fd9887d9a..57d4fbc59f83 100644
--- a/lib/ProfileData/InstrProf.cpp
+++ b/lib/ProfileData/InstrProf.cpp
@@ -478,7 +478,7 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
return Error::success();
}
-void InstrProfRecord::accumuateCounts(CountSumOrPercent &Sum) const {
+void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
for (size_t F = 0, E = Counts.size(); F < E; ++F)
@@ -552,7 +552,7 @@ void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
uint64_t ValueCutoff) {
// FuncLevel CountSum for other should already computed and nonzero.
assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
- accumuateCounts(FuncLevelOverlap.Base);
+ accumulateCounts(FuncLevelOverlap.Base);
bool Mismatch = (Counts.size() != Other.Counts.size());
// Check if the value profiles mismatch.
@@ -1078,12 +1078,10 @@ bool isIRPGOFlagSet(const Module *M) {
if (!IRInstrVar->hasInitializer())
return false;
- const Constant *InitVal = IRInstrVar->getInitializer();
+ auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
if (!InitVal)
return false;
-
- return (dyn_cast<ConstantInt>(InitVal)->getZExtValue() &
- VARIANT_MASK_IR_PROF) != 0;
+ return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
}
// Check if we can safely rename this Comdat function.
@@ -1166,9 +1164,9 @@ void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
}
}
-Error OverlapStats::accumuateCounts(const std::string &BaseFilename,
- const std::string &TestFilename,
- bool IsCS) {
+Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
+ const std::string &TestFilename,
+ bool IsCS) {
auto getProfileSum = [IsCS](const std::string &Filename,
CountSumOrPercent &Sum) -> Error {
auto ReaderOrErr = InstrProfReader::create(Filename);
@@ -1176,7 +1174,7 @@ Error OverlapStats::accumuateCounts(const std::string &BaseFilename,
return E;
}
auto Reader = std::move(ReaderOrErr.get());
- Reader->accumuateCounts(Sum, IsCS);
+ Reader->accumulateCounts(Sum, IsCS);
return Error::success();
};
auto Ret = getProfileSum(BaseFilename, Base);
diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp
index fec1c152991c..23d078a3ddee 100644
--- a/lib/ProfileData/InstrProfReader.cpp
+++ b/lib/ProfileData/InstrProfReader.cpp
@@ -119,7 +119,7 @@ IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
// Create the reader.
if (!IndexedInstrProfReader::hasFormat(*Buffer))
return make_error<InstrProfError>(instrprof_error::bad_magic);
- auto Result = llvm::make_unique<IndexedInstrProfReader>(
+ auto Result = std::make_unique<IndexedInstrProfReader>(
std::move(Buffer), std::move(RemappingBuffer));
// Initialize the reader and return the result.
@@ -385,7 +385,7 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
NamesStart = Start + NamesOffset;
ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
- std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
+ std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
if (Error E = createSymtab(*NewSymtab.get()))
return E;
@@ -413,13 +413,19 @@ Error RawInstrProfReader<IntPtrT>::readRawCounts(
if (NumCounters == 0)
return error(instrprof_error::malformed);
- auto RawCounts = makeArrayRef(getCounter(CounterPtr), NumCounters);
auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart);
+ ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart;
- // Check bounds.
- if (RawCounts.data() < CountersStart ||
- RawCounts.data() + RawCounts.size() > NamesStartAsCounter)
+ // Check bounds. Note that the counter pointer embedded in the data record
+ // may itself be corrupt.
+ if (NumCounters > MaxNumCounters)
return error(instrprof_error::malformed);
+ ptrdiff_t CounterOffset = getCounterOffset(CounterPtr);
+ if (CounterOffset < 0 || CounterOffset > MaxNumCounters ||
+ (CounterOffset + NumCounters) > MaxNumCounters)
+ return error(instrprof_error::malformed);
+
+ auto RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters);
if (ShouldSwapBytes) {
Record.Counts.clear();
@@ -767,7 +773,7 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
UseCS ? this->CS_Summary : this->Summary;
// initialize InstrProfSummary using the SummaryData from disk.
- Summary = llvm::make_unique<ProfileSummary>(
+ Summary = std::make_unique<ProfileSummary>(
UseCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr,
DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
SummaryData->get(Summary::MaxBlockCount),
@@ -777,13 +783,13 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
SummaryData->get(Summary::TotalNumFunctions));
return Cur + SummarySize;
} else {
- // For older version of profile data, we need to compute on the fly:
- using namespace IndexedInstrProf;
-
+ // The older versions do not support a profile summary. This just computes
+ // an empty summary, which will not result in accurate hot/cold detection.
+ // We would need to call addRecord for all NamedInstrProfRecords to get the
+ // correct summary. However, this version is old (prior to early 2016) and
+ // has not been supporting an accurate summary for several years.
InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
- // FIXME: This only computes an empty summary. Need to call addRecord for
- // all NamedInstrProfRecords to get the correct summary.
- this->Summary = Builder.getSummary();
+ Summary = Builder.getSummary();
return Cur;
}
}
@@ -827,18 +833,18 @@ Error IndexedInstrProfReader::readHeader() {
// The rest of the file is an on disk hash table.
auto IndexPtr =
- llvm::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
+ std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
Start + HashOffset, Cur, Start, HashType, FormatVersion);
// Load the remapping table now if requested.
if (RemappingBuffer) {
- Remapper = llvm::make_unique<
+ Remapper = std::make_unique<
InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
std::move(RemappingBuffer), *IndexPtr);
if (Error E = Remapper->populateRemappings())
return E;
} else {
- Remapper = llvm::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
+ Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
}
Index = std::move(IndexPtr);
@@ -849,7 +855,7 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
if (Symtab.get())
return *Symtab.get();
- std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
+ std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
if (Error E = Index->populateSymtab(*NewSymtab.get())) {
consumeError(error(InstrProfError::take(std::move(E))));
}
@@ -901,7 +907,7 @@ Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
return success();
}
-void InstrProfReader::accumuateCounts(CountSumOrPercent &Sum, bool IsCS) {
+void InstrProfReader::accumulateCounts(CountSumOrPercent &Sum, bool IsCS) {
uint64_t NumFuncs = 0;
for (const auto &Func : *this) {
if (isIRLevelProfile()) {
@@ -909,7 +915,7 @@ void InstrProfReader::accumuateCounts(CountSumOrPercent &Sum, bool IsCS) {
if (FuncIsCS != IsCS)
continue;
}
- Func.accumuateCounts(Sum);
+ Func.accumulateCounts(Sum);
++NumFuncs;
}
Sum.NumEntries = NumFuncs;
diff --git a/lib/ProfileData/InstrProfWriter.cpp b/lib/ProfileData/InstrProfWriter.cpp
index 4ca2defd26da..ccb270e0b719 100644
--- a/lib/ProfileData/InstrProfWriter.cpp
+++ b/lib/ProfileData/InstrProfWriter.cpp
@@ -193,7 +193,7 @@ void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
const OverlapFuncFilters &FuncFilter) {
auto Name = Other.Name;
auto Hash = Other.Hash;
- Other.accumuateCounts(FuncLevelOverlap.Test);
+ Other.accumulateCounts(FuncLevelOverlap.Test);
if (FunctionData.find(Name) == FunctionData.end()) {
Overlap.addOneUnique(FuncLevelOverlap.Test);
return;
diff --git a/lib/ProfileData/ProfileSummaryBuilder.cpp b/lib/ProfileData/ProfileSummaryBuilder.cpp
index 4d5b00935742..3299b5f92069 100644
--- a/lib/ProfileData/ProfileSummaryBuilder.cpp
+++ b/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -93,14 +93,14 @@ void ProfileSummaryBuilder::computeDetailedSummary() {
std::unique_ptr<ProfileSummary> SampleProfileSummaryBuilder::getSummary() {
computeDetailedSummary();
- return llvm::make_unique<ProfileSummary>(
+ return std::make_unique<ProfileSummary>(
ProfileSummary::PSK_Sample, DetailedSummary, TotalCount, MaxCount, 0,
MaxFunctionCount, NumCounts, NumFunctions);
}
std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() {
computeDetailedSummary();
- return llvm::make_unique<ProfileSummary>(
+ return std::make_unique<ProfileSummary>(
ProfileSummary::PSK_Instr, DetailedSummary, TotalCount, MaxCount,
MaxInternalBlockCount, MaxFunctionCount, NumCounts, NumFunctions);
}
diff --git a/lib/ProfileData/SampleProf.cpp b/lib/ProfileData/SampleProf.cpp
index e17865cd15a4..003e8d4d4296 100644
--- a/lib/ProfileData/SampleProf.cpp
+++ b/lib/ProfileData/SampleProf.cpp
@@ -16,7 +16,9 @@
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/LEB128.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/raw_ostream.h"
#include <string>
@@ -28,8 +30,6 @@ using namespace sampleprof;
namespace llvm {
namespace sampleprof {
SampleProfileFormat FunctionSamples::Format;
-DenseMap<uint64_t, StringRef> FunctionSamples::GUIDToFuncNameMap;
-Module *FunctionSamples::CurrentModule;
} // namespace sampleprof
} // namespace llvm
@@ -68,6 +68,12 @@ class SampleProfErrorCategoryType : public std::error_category {
return "Counter overflow";
case sampleprof_error::ostream_seek_unsupported:
return "Ostream does not support seek";
+ case sampleprof_error::compress_failed:
+ return "Compress failure";
+ case sampleprof_error::uncompress_failed:
+ return "Uncompress failure";
+ case sampleprof_error::zlib_unavailable:
+ return "Zlib is unavailable";
}
llvm_unreachable("A value of sampleprof_error has no message.");
}
@@ -102,8 +108,8 @@ void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
OS << NumSamples;
if (hasCalls()) {
OS << ", calls:";
- for (const auto &I : getCallTargets())
- OS << " " << I.first() << ":" << I.second;
+ for (const auto &I : getSortedCallTargets())
+ OS << " " << I.first << ":" << I.second;
}
OS << "\n";
}
@@ -149,6 +155,7 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
FS.second.print(OS, Indent + 4);
}
}
+ OS.indent(Indent);
OS << "}\n";
} else {
OS << "No inlined callsites in this function\n";
@@ -190,3 +197,44 @@ FunctionSamples::findFunctionSamples(const DILocation *DIL) const {
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void FunctionSamples::dump() const { print(dbgs(), 0); }
#endif
+
+std::error_code ProfileSymbolList::read(const uint8_t *Data,
+ uint64_t ListSize) {
+ const char *ListStart = reinterpret_cast<const char *>(Data);
+ uint64_t Size = 0;
+ while (Size < ListSize) {
+ StringRef Str(ListStart + Size);
+ add(Str);
+ Size += Str.size() + 1;
+ }
+ if (Size != ListSize)
+ return sampleprof_error::malformed;
+ return sampleprof_error::success;
+}
+
+std::error_code ProfileSymbolList::write(raw_ostream &OS) {
+ // Sort the symbols before output. If doing compression.
+ // It will make the compression much more effective.
+ std::vector<StringRef> SortedList;
+ SortedList.insert(SortedList.begin(), Syms.begin(), Syms.end());
+ llvm::sort(SortedList);
+
+ std::string OutputString;
+ for (auto &Sym : SortedList) {
+ OutputString.append(Sym.str());
+ OutputString.append(1, '\0');
+ }
+
+ OS << OutputString;
+ return sampleprof_error::success;
+}
+
+void ProfileSymbolList::dump(raw_ostream &OS) const {
+ OS << "======== Dump profile symbol list ========\n";
+ std::vector<StringRef> SortedList;
+ SortedList.insert(SortedList.begin(), Syms.begin(), Syms.end());
+ llvm::sort(SortedList);
+
+ for (auto &Sym : SortedList)
+ OS << Sym << "\n";
+}
diff --git a/lib/ProfileData/SampleProfReader.cpp b/lib/ProfileData/SampleProfReader.cpp
index 192b6c711562..001aafce7bfd 100644
--- a/lib/ProfileData/SampleProfReader.cpp
+++ b/lib/ProfileData/SampleProfReader.cpp
@@ -26,6 +26,7 @@
#include "llvm/IR/ProfileSummary.h"
#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/LineIterator.h"
@@ -190,7 +191,7 @@ static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth,
/// the expected format.
///
/// \returns true if the file was loaded successfully, false otherwise.
-std::error_code SampleProfileReaderText::read() {
+std::error_code SampleProfileReaderText::readImpl() {
line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
sampleprof_error Result = sampleprof_error::success;
@@ -345,7 +346,7 @@ inline ErrorOr<uint32_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
return *Idx;
}
-ErrorOr<StringRef> SampleProfileReaderRawBinary::readStringFromTable() {
+ErrorOr<StringRef> SampleProfileReaderBinary::readStringFromTable() {
auto Idx = readStringIndex(NameTable);
if (std::error_code EC = Idx.getError())
return EC;
@@ -438,7 +439,9 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
return sampleprof_error::success;
}
-std::error_code SampleProfileReaderBinary::readFuncProfile() {
+std::error_code
+SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
+ Data = Start;
auto NumHeadSamples = readNumber<uint64_t>();
if (std::error_code EC = NumHeadSamples.getError())
return EC;
@@ -458,25 +461,210 @@ std::error_code SampleProfileReaderBinary::readFuncProfile() {
return sampleprof_error::success;
}
-std::error_code SampleProfileReaderBinary::read() {
+std::error_code SampleProfileReaderBinary::readImpl() {
while (!at_eof()) {
- if (std::error_code EC = readFuncProfile())
+ if (std::error_code EC = readFuncProfile(Data))
+ return EC;
+ }
+
+ return sampleprof_error::success;
+}
+
+std::error_code
+SampleProfileReaderExtBinary::readOneSection(const uint8_t *Start,
+ uint64_t Size, SecType Type) {
+ Data = Start;
+ End = Start + Size;
+ switch (Type) {
+ case SecProfSummary:
+ if (std::error_code EC = readSummary())
+ return EC;
+ break;
+ case SecNameTable:
+ if (std::error_code EC = readNameTable())
+ return EC;
+ break;
+ case SecLBRProfile:
+ if (std::error_code EC = readFuncProfiles())
+ return EC;
+ break;
+ case SecProfileSymbolList:
+ if (std::error_code EC = readProfileSymbolList())
return EC;
+ break;
+ case SecFuncOffsetTable:
+ if (std::error_code EC = readFuncOffsetTable())
+ return EC;
+ break;
+ default:
+ break;
}
+ return sampleprof_error::success;
+}
+
+void SampleProfileReaderExtBinary::collectFuncsFrom(const Module &M) {
+ UseAllFuncs = false;
+ FuncsToUse.clear();
+ for (auto &F : M)
+ FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));
+}
+
+std::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() {
+ auto Size = readNumber<uint64_t>();
+ if (std::error_code EC = Size.getError())
+ return EC;
+
+ FuncOffsetTable.reserve(*Size);
+ for (uint32_t I = 0; I < *Size; ++I) {
+ auto FName(readStringFromTable());
+ if (std::error_code EC = FName.getError())
+ return EC;
+ auto Offset = readNumber<uint64_t>();
+ if (std::error_code EC = Offset.getError())
+ return EC;
+
+ FuncOffsetTable[*FName] = *Offset;
+ }
return sampleprof_error::success;
}
-std::error_code SampleProfileReaderCompactBinary::read() {
- for (auto Name : FuncsToUse) {
- auto GUID = std::to_string(MD5Hash(Name));
- auto iter = FuncOffsetTable.find(StringRef(GUID));
- if (iter == FuncOffsetTable.end())
+std::error_code SampleProfileReaderExtBinary::readFuncProfiles() {
+ const uint8_t *Start = Data;
+ if (UseAllFuncs) {
+ while (Data < End) {
+ if (std::error_code EC = readFuncProfile(Data))
+ return EC;
+ }
+ assert(Data == End && "More data is read than expected");
+ return sampleprof_error::success;
+ }
+
+ if (Remapper) {
+ for (auto Name : FuncsToUse) {
+ Remapper->insert(Name);
+ }
+ }
+
+ for (auto NameOffset : FuncOffsetTable) {
+ auto FuncName = NameOffset.first;
+ if (!FuncsToUse.count(FuncName) &&
+ (!Remapper || !Remapper->exist(FuncName)))
continue;
+ const uint8_t *FuncProfileAddr = Start + NameOffset.second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
+
+ Data = End;
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinary::readProfileSymbolList() {
+ if (!ProfSymList)
+ ProfSymList = std::make_unique<ProfileSymbolList>();
+
+ if (std::error_code EC = ProfSymList->read(Data, End - Data))
+ return EC;
+
+ Data = End;
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
+ const uint8_t *SecStart, const uint64_t SecSize,
+ const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
+ Data = SecStart;
+ End = SecStart + SecSize;
+ auto DecompressSize = readNumber<uint64_t>();
+ if (std::error_code EC = DecompressSize.getError())
+ return EC;
+ DecompressBufSize = *DecompressSize;
+
+ auto CompressSize = readNumber<uint64_t>();
+ if (std::error_code EC = CompressSize.getError())
+ return EC;
+
+ if (!llvm::zlib::isAvailable())
+ return sampleprof_error::zlib_unavailable;
+
+ StringRef CompressedStrings(reinterpret_cast<const char *>(Data),
+ *CompressSize);
+ char *Buffer = Allocator.Allocate<char>(DecompressBufSize);
+ size_t UCSize = DecompressBufSize;
+ llvm::Error E =
+ zlib::uncompress(CompressedStrings, Buffer, UCSize);
+ if (E)
+ return sampleprof_error::uncompress_failed;
+ DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
+ const uint8_t *BufStart =
+ reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
+
+ for (auto &Entry : SecHdrTable) {
+ // Skip empty section.
+ if (!Entry.Size)
+ continue;
+
+ const uint8_t *SecStart = BufStart + Entry.Offset;
+ uint64_t SecSize = Entry.Size;
+
+ // If the section is compressed, decompress it into a buffer
+ // DecompressBuf before reading the actual data. The pointee of
+ // 'Data' will be changed to buffer hold by DecompressBuf
+ // temporarily when reading the actual data.
+ bool isCompressed = hasSecFlag(Entry, SecFlagCompress);
+ if (isCompressed) {
+ const uint8_t *DecompressBuf;
+ uint64_t DecompressBufSize;
+ if (std::error_code EC = decompressSection(
+ SecStart, SecSize, DecompressBuf, DecompressBufSize))
+ return EC;
+ SecStart = DecompressBuf;
+ SecSize = DecompressBufSize;
+ }
+
+ if (std::error_code EC = readOneSection(SecStart, SecSize, Entry.Type))
+ return EC;
+ if (Data != SecStart + SecSize)
+ return sampleprof_error::malformed;
+
+ // Change the pointee of 'Data' from DecompressBuf to original Buffer.
+ if (isCompressed) {
+ Data = BufStart + Entry.Offset;
+ End = BufStart + Buffer->getBufferSize();
+ }
+ }
+
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderCompactBinary::readImpl() {
+ std::vector<uint64_t> OffsetsToUse;
+ if (UseAllFuncs) {
+ for (auto FuncEntry : FuncOffsetTable) {
+ OffsetsToUse.push_back(FuncEntry.second);
+ }
+ }
+ else {
+ for (auto Name : FuncsToUse) {
+ auto GUID = std::to_string(MD5Hash(Name));
+ auto iter = FuncOffsetTable.find(StringRef(GUID));
+ if (iter == FuncOffsetTable.end())
+ continue;
+ OffsetsToUse.push_back(iter->second);
+ }
+ }
+
+ for (auto Offset : OffsetsToUse) {
const uint8_t *SavedData = Data;
- Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
- iter->second;
- if (std::error_code EC = readFuncProfile())
+ if (std::error_code EC = readFuncProfile(
+ reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
+ Offset))
return EC;
Data = SavedData;
}
@@ -489,6 +677,12 @@ std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
return sampleprof_error::bad_magic;
}
+std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
+ if (Magic == SPMagic(SPF_Ext_Binary))
+ return sampleprof_error::success;
+ return sampleprof_error::bad_magic;
+}
+
std::error_code
SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) {
if (Magic == SPMagic(SPF_Compact_Binary))
@@ -496,7 +690,7 @@ SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) {
return sampleprof_error::bad_magic;
}
-std::error_code SampleProfileReaderRawBinary::readNameTable() {
+std::error_code SampleProfileReaderBinary::readNameTable() {
auto Size = readNumber<uint32_t>();
if (std::error_code EC = Size.getError())
return EC;
@@ -525,10 +719,98 @@ std::error_code SampleProfileReaderCompactBinary::readNameTable() {
return sampleprof_error::success;
}
-std::error_code SampleProfileReaderBinary::readHeader() {
- Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
- End = Data + Buffer->getBufferSize();
+std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTableEntry() {
+ SecHdrTableEntry Entry;
+ auto Type = readUnencodedNumber<uint64_t>();
+ if (std::error_code EC = Type.getError())
+ return EC;
+ Entry.Type = static_cast<SecType>(*Type);
+ auto Flags = readUnencodedNumber<uint64_t>();
+ if (std::error_code EC = Flags.getError())
+ return EC;
+ Entry.Flags = *Flags;
+
+ auto Offset = readUnencodedNumber<uint64_t>();
+ if (std::error_code EC = Offset.getError())
+ return EC;
+ Entry.Offset = *Offset;
+
+ auto Size = readUnencodedNumber<uint64_t>();
+ if (std::error_code EC = Size.getError())
+ return EC;
+ Entry.Size = *Size;
+
+ SecHdrTable.push_back(std::move(Entry));
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
+ auto EntryNum = readUnencodedNumber<uint64_t>();
+ if (std::error_code EC = EntryNum.getError())
+ return EC;
+
+ for (uint32_t i = 0; i < (*EntryNum); i++)
+ if (std::error_code EC = readSecHdrTableEntry())
+ return EC;
+
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
+ const uint8_t *BufStart =
+ reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
+ Data = BufStart;
+ End = BufStart + Buffer->getBufferSize();
+
+ if (std::error_code EC = readMagicIdent())
+ return EC;
+
+ if (std::error_code EC = readSecHdrTable())
+ return EC;
+
+ return sampleprof_error::success;
+}
+
+uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
+ for (auto &Entry : SecHdrTable) {
+ if (Entry.Type == Type)
+ return Entry.Size;
+ }
+ return 0;
+}
+
+uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
+ // Sections in SecHdrTable is not necessarily in the same order as
+ // sections in the profile because section like FuncOffsetTable needs
+ // to be written after section LBRProfile but needs to be read before
+ // section LBRProfile, so we cannot simply use the last entry in
+ // SecHdrTable to calculate the file size.
+ uint64_t FileSize = 0;
+ for (auto &Entry : SecHdrTable) {
+ FileSize = std::max(Entry.Offset + Entry.Size, FileSize);
+ }
+ return FileSize;
+}
+
+bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
+ uint64_t TotalSecsSize = 0;
+ for (auto &Entry : SecHdrTable) {
+ OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset
+ << ", Size: " << Entry.Size << "\n";
+ TotalSecsSize += getSectionSize(Entry.Type);
+ }
+ uint64_t HeaderSize = SecHdrTable.front().Offset;
+ assert(HeaderSize + TotalSecsSize == getFileSize() &&
+ "Size of 'header + sections' doesn't match the total size of profile");
+
+ OS << "Header Size: " << HeaderSize << "\n";
+ OS << "Total Sections Size: " << TotalSecsSize << "\n";
+ OS << "File Size: " << getFileSize() << "\n";
+ return true;
+}
+
+std::error_code SampleProfileReaderBinary::readMagicIdent() {
// Read and check the magic identifier.
auto Magic = readNumber<uint64_t>();
if (std::error_code EC = Magic.getError())
@@ -543,6 +825,16 @@ std::error_code SampleProfileReaderBinary::readHeader() {
else if (*Version != SPVersion())
return sampleprof_error::unsupported_version;
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderBinary::readHeader() {
+ Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
+ End = Data + Buffer->getBufferSize();
+
+ if (std::error_code EC = readMagicIdent())
+ return EC;
+
if (std::error_code EC = readSummary())
return EC;
@@ -590,12 +882,11 @@ std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {
return sampleprof_error::success;
}
-void SampleProfileReaderCompactBinary::collectFuncsToUse(const Module &M) {
+void SampleProfileReaderCompactBinary::collectFuncsFrom(const Module &M) {
+ UseAllFuncs = false;
FuncsToUse.clear();
- for (auto &F : M) {
- StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
- FuncsToUse.insert(CanonName);
- }
+ for (auto &F : M)
+ FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));
}
std::error_code SampleProfileReaderBinary::readSummaryEntry(
@@ -647,7 +938,7 @@ std::error_code SampleProfileReaderBinary::readSummary() {
if (EC != sampleprof_error::success)
return EC;
}
- Summary = llvm::make_unique<ProfileSummary>(
+ Summary = std::make_unique<ProfileSummary>(
ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,
*MaxFunctionCount, *NumBlocks, *NumFunctions);
@@ -661,6 +952,13 @@ bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
return Magic == SPMagic();
}
+bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
+ const uint8_t *Data =
+ reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
+ uint64_t Magic = decodeULEB128(Data);
+ return Magic == SPMagic(SPF_Ext_Binary);
+}
+
bool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) {
const uint8_t *Data =
reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
@@ -894,7 +1192,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
///
/// This format is generated by the Linux Perf conversion tool at
/// https://github.com/google/autofdo.
-std::error_code SampleProfileReaderGCC::read() {
+std::error_code SampleProfileReaderGCC::readImpl() {
// Read the string table.
if (std::error_code EC = readNameTable())
return EC;
@@ -911,38 +1209,31 @@ bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
return Magic == "adcg*704";
}
-std::error_code SampleProfileReaderItaniumRemapper::read() {
- // If the underlying data is in compact format, we can't remap it because
+void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
+ // If the reader is in compact format, we can't remap it because
// we don't know what the original function names were.
- if (getFormat() == SPF_Compact_Binary) {
+ if (Reader.getFormat() == SPF_Compact_Binary) {
Ctx.diagnose(DiagnosticInfoSampleProfile(
- Buffer->getBufferIdentifier(),
+ Reader.getBuffer()->getBufferIdentifier(),
"Profile data remapping cannot be applied to profile data "
"in compact format (original mangled names are not available).",
DS_Warning));
- return sampleprof_error::success;
- }
-
- if (Error E = Remappings.read(*Buffer)) {
- handleAllErrors(
- std::move(E), [&](const SymbolRemappingParseError &ParseError) {
- reportError(ParseError.getLineNum(), ParseError.getMessage());
- });
- return sampleprof_error::malformed;
+ return;
}
- for (auto &Sample : getProfiles())
- if (auto Key = Remappings.insert(Sample.first()))
+ assert(Remappings && "should be initialized while creating remapper");
+ for (auto &Sample : Reader.getProfiles())
+ if (auto Key = Remappings->insert(Sample.first()))
SampleMap.insert({Key, &Sample.second});
- return sampleprof_error::success;
+ RemappingApplied = true;
}
FunctionSamples *
SampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) {
- if (auto Key = Remappings.lookup(Fname))
+ if (auto Key = Remappings->lookup(Fname))
return SampleMap.lookup(Key);
- return SampleProfileReader::getSamplesFor(Fname);
+ return nullptr;
}
/// Prepare a memory buffer for the contents of \p Filename.
@@ -968,13 +1259,16 @@ setupMemoryBuffer(const Twine &Filename) {
///
/// \param C The LLVM context to use to emit diagnostics.
///
+/// \param RemapFilename The file used for profile remapping.
+///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReader>>
-SampleProfileReader::create(const Twine &Filename, LLVMContext &C) {
+SampleProfileReader::create(const std::string Filename, LLVMContext &C,
+ const std::string RemapFilename) {
auto BufferOrError = setupMemoryBuffer(Filename);
if (std::error_code EC = BufferOrError.getError())
return EC;
- return create(BufferOrError.get(), C);
+ return create(BufferOrError.get(), C, RemapFilename);
}
/// Create a sample profile remapper from the given input, to remap the
@@ -982,20 +1276,48 @@ SampleProfileReader::create(const Twine &Filename, LLVMContext &C) {
///
/// \param Filename The file to open.
///
-/// \param C The LLVM context to use to emit diagnostics.
+/// \param Reader The profile reader the remapper is going to be applied to.
///
-/// \param Underlying The underlying profile data reader to remap.
+/// \param C The LLVM context to use to emit diagnostics.
///
/// \returns an error code indicating the status of the created reader.
-ErrorOr<std::unique_ptr<SampleProfileReader>>
-SampleProfileReaderItaniumRemapper::create(
- const Twine &Filename, LLVMContext &C,
- std::unique_ptr<SampleProfileReader> Underlying) {
+ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
+SampleProfileReaderItaniumRemapper::create(const std::string Filename,
+ SampleProfileReader &Reader,
+ LLVMContext &C) {
auto BufferOrError = setupMemoryBuffer(Filename);
if (std::error_code EC = BufferOrError.getError())
return EC;
- return llvm::make_unique<SampleProfileReaderItaniumRemapper>(
- std::move(BufferOrError.get()), C, std::move(Underlying));
+ return create(BufferOrError.get(), Reader, C);
+}
+
+/// Create a sample profile remapper from the given input, to remap the
+/// function names in the given profile data.
+///
+/// \param B The memory buffer to create the reader from (assumes ownership).
+///
+/// \param C The LLVM context to use to emit diagnostics.
+///
+/// \param Reader The profile reader the remapper is going to be applied to.
+///
+/// \returns an error code indicating the status of the created reader.
+ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
+SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
+ SampleProfileReader &Reader,
+ LLVMContext &C) {
+ auto Remappings = std::make_unique<SymbolRemappingReader>();
+ if (Error E = Remappings->read(*B.get())) {
+ handleAllErrors(
+ std::move(E), [&](const SymbolRemappingParseError &ParseError) {
+ C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
+ ParseError.getLineNum(),
+ ParseError.getMessage()));
+ });
+ return sampleprof_error::malformed;
+ }
+
+ return std::make_unique<SampleProfileReaderItaniumRemapper>(
+ std::move(B), std::move(Remappings), Reader);
}
/// Create a sample profile reader based on the format of the input data.
@@ -1004,12 +1326,17 @@ SampleProfileReaderItaniumRemapper::create(
///
/// \param C The LLVM context to use to emit diagnostics.
///
+/// \param RemapFilename The file used for profile remapping.
+///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReader>>
-SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
+SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
+ const std::string RemapFilename) {
std::unique_ptr<SampleProfileReader> Reader;
if (SampleProfileReaderRawBinary::hasFormat(*B))
Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));
+ else if (SampleProfileReaderExtBinary::hasFormat(*B))
+ Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));
else if (SampleProfileReaderCompactBinary::hasFormat(*B))
Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C));
else if (SampleProfileReaderGCC::hasFormat(*B))
@@ -1019,9 +1346,21 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
else
return sampleprof_error::unrecognized_format;
+ if (!RemapFilename.empty()) {
+ auto ReaderOrErr =
+ SampleProfileReaderItaniumRemapper::create(RemapFilename, *Reader, C);
+ if (std::error_code EC = ReaderOrErr.getError()) {
+ std::string Msg = "Could not create remapper: " + EC.message();
+ C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg));
+ return EC;
+ }
+ Reader->Remapper = std::move(ReaderOrErr.get());
+ }
+
FunctionSamples::Format = Reader->getFormat();
- if (std::error_code EC = Reader->readHeader())
+ if (std::error_code EC = Reader->readHeader()) {
return EC;
+ }
return std::move(Reader);
}
diff --git a/lib/ProfileData/SampleProfWriter.cpp b/lib/ProfileData/SampleProfWriter.cpp
index 8b876e0aa5d9..8d09af31f94b 100644
--- a/lib/ProfileData/SampleProfWriter.cpp
+++ b/lib/ProfileData/SampleProfWriter.cpp
@@ -21,6 +21,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/EndianStream.h"
#include "llvm/Support/ErrorOr.h"
@@ -39,11 +40,8 @@
using namespace llvm;
using namespace sampleprof;
-std::error_code
-SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
- if (std::error_code EC = writeHeader(ProfileMap))
- return EC;
-
+std::error_code SampleProfileWriter::writeFuncProfiles(
+ const StringMap<FunctionSamples> &ProfileMap) {
// Sort the ProfileMap by total samples.
typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples;
std::vector<NameFunctionSamples> V;
@@ -58,12 +56,161 @@ SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
});
for (const auto &I : V) {
- if (std::error_code EC = write(*I.second))
+ if (std::error_code EC = writeSample(*I.second))
return EC;
}
return sampleprof_error::success;
}
+std::error_code
+SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
+ if (std::error_code EC = writeHeader(ProfileMap))
+ return EC;
+
+ if (std::error_code EC = writeFuncProfiles(ProfileMap))
+ return EC;
+
+ return sampleprof_error::success;
+}
+
+SecHdrTableEntry &
+SampleProfileWriterExtBinaryBase::getEntryInLayout(SecType Type) {
+ auto SecIt = std::find_if(
+ SectionHdrLayout.begin(), SectionHdrLayout.end(),
+ [=](const auto &Entry) -> bool { return Entry.Type == Type; });
+ return *SecIt;
+}
+
+/// Return the current position and prepare to use it as the start
+/// position of a section.
+uint64_t SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type) {
+ uint64_t SectionStart = OutputStream->tell();
+ auto &Entry = getEntryInLayout(Type);
+ // Use LocalBuf as a temporary output for writting data.
+ if (hasSecFlag(Entry, SecFlagCompress))
+ LocalBufStream.swap(OutputStream);
+ return SectionStart;
+}
+
+std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
+ if (!llvm::zlib::isAvailable())
+ return sampleprof_error::zlib_unavailable;
+ std::string &UncompressedStrings =
+ static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
+ if (UncompressedStrings.size() == 0)
+ return sampleprof_error::success;
+ auto &OS = *OutputStream;
+ SmallString<128> CompressedStrings;
+ llvm::Error E = zlib::compress(UncompressedStrings, CompressedStrings,
+ zlib::BestSizeCompression);
+ if (E)
+ return sampleprof_error::compress_failed;
+ encodeULEB128(UncompressedStrings.size(), OS);
+ encodeULEB128(CompressedStrings.size(), OS);
+ OS << CompressedStrings.str();
+ UncompressedStrings.clear();
+ return sampleprof_error::success;
+}
+
+/// Add a new section into section header table.
+std::error_code
+SampleProfileWriterExtBinaryBase::addNewSection(SecType Type,
+ uint64_t SectionStart) {
+ auto Entry = getEntryInLayout(Type);
+ if (hasSecFlag(Entry, SecFlagCompress)) {
+ LocalBufStream.swap(OutputStream);
+ if (std::error_code EC = compressAndOutput())
+ return EC;
+ }
+ SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
+ OutputStream->tell() - SectionStart});
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileWriterExtBinaryBase::write(
+ const StringMap<FunctionSamples> &ProfileMap) {
+ if (std::error_code EC = writeHeader(ProfileMap))
+ return EC;
+
+ std::string LocalBuf;
+ LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
+ if (std::error_code EC = writeSections(ProfileMap))
+ return EC;
+
+ if (std::error_code EC = writeSecHdrTable())
+ return EC;
+
+ return sampleprof_error::success;
+}
+
+std::error_code
+SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) {
+ uint64_t Offset = OutputStream->tell();
+ StringRef Name = S.getName();
+ FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
+ encodeULEB128(S.getHeadSamples(), *OutputStream);
+ return writeBody(S);
+}
+
+std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() {
+ auto &OS = *OutputStream;
+
+ // Write out the table size.
+ encodeULEB128(FuncOffsetTable.size(), OS);
+
+ // Write out FuncOffsetTable.
+ for (auto entry : FuncOffsetTable) {
+ writeNameIdx(entry.first);
+ encodeULEB128(entry.second, OS);
+ }
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileWriterExtBinary::writeSections(
+ const StringMap<FunctionSamples> &ProfileMap) {
+ uint64_t SectionStart = markSectionStart(SecProfSummary);
+ computeSummary(ProfileMap);
+ if (auto EC = writeSummary())
+ return EC;
+ if (std::error_code EC = addNewSection(SecProfSummary, SectionStart))
+ return EC;
+
+ // Generate the name table for all the functions referenced in the profile.
+ SectionStart = markSectionStart(SecNameTable);
+ for (const auto &I : ProfileMap) {
+ addName(I.first());
+ addNames(I.second);
+ }
+ writeNameTable();
+ if (std::error_code EC = addNewSection(SecNameTable, SectionStart))
+ return EC;
+
+ SectionStart = markSectionStart(SecLBRProfile);
+ SecLBRProfileStart = OutputStream->tell();
+ if (std::error_code EC = writeFuncProfiles(ProfileMap))
+ return EC;
+ if (std::error_code EC = addNewSection(SecLBRProfile, SectionStart))
+ return EC;
+
+ if (ProfSymList && ProfSymList->toCompress())
+ setToCompressSection(SecProfileSymbolList);
+
+ SectionStart = markSectionStart(SecProfileSymbolList);
+ if (ProfSymList && ProfSymList->size() > 0)
+ if (std::error_code EC = ProfSymList->write(*OutputStream))
+ return EC;
+ if (std::error_code EC = addNewSection(SecProfileSymbolList, SectionStart))
+ return EC;
+
+ SectionStart = markSectionStart(SecFuncOffsetTable);
+ if (std::error_code EC = writeFuncOffsetTable())
+ return EC;
+ if (std::error_code EC = addNewSection(SecFuncOffsetTable, SectionStart))
+ return EC;
+
+ return sampleprof_error::success;
+}
+
std::error_code SampleProfileWriterCompactBinary::write(
const StringMap<FunctionSamples> &ProfileMap) {
if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
@@ -81,7 +228,7 @@ std::error_code SampleProfileWriterCompactBinary::write(
///
/// The format used here is more structured and deliberate because
/// it needs to be parsed by the SampleProfileReaderText class.
-std::error_code SampleProfileWriterText::write(const FunctionSamples &S) {
+std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
auto &OS = *OutputStream;
OS << S.getName() << ":" << S.getTotalSamples();
if (Indent == 0)
@@ -100,8 +247,8 @@ std::error_code SampleProfileWriterText::write(const FunctionSamples &S) {
OS << Sample.getSamples();
- for (const auto &J : Sample.getCallTargets())
- OS << " " << J.first() << ":" << J.second;
+ for (const auto &J : Sample.getSortedCallTargets())
+ OS << " " << J.first << ":" << J.second;
OS << "\n";
}
@@ -117,7 +264,7 @@ std::error_code SampleProfileWriterText::write(const FunctionSamples &S) {
OS << Loc.LineOffset << ": ";
else
OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
- if (std::error_code EC = write(CalleeSamples))
+ if (std::error_code EC = writeSample(CalleeSamples))
return EC;
}
Indent -= 1;
@@ -163,7 +310,7 @@ void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) {
NameTable[N] = i++;
}
-std::error_code SampleProfileWriterRawBinary::writeNameTable() {
+std::error_code SampleProfileWriterBinary::writeNameTable() {
auto &OS = *OutputStream;
std::set<StringRef> V;
stablizeNameTable(V);
@@ -214,25 +361,18 @@ std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
return sampleprof_error::success;
}
-std::error_code SampleProfileWriterRawBinary::writeMagicIdent() {
- auto &OS = *OutputStream;
- // Write file magic identifier.
- encodeULEB128(SPMagic(), OS);
- encodeULEB128(SPVersion(), OS);
- return sampleprof_error::success;
-}
-
-std::error_code SampleProfileWriterCompactBinary::writeMagicIdent() {
+std::error_code
+SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
auto &OS = *OutputStream;
// Write file magic identifier.
- encodeULEB128(SPMagic(SPF_Compact_Binary), OS);
+ encodeULEB128(SPMagic(Format), OS);
encodeULEB128(SPVersion(), OS);
return sampleprof_error::success;
}
std::error_code SampleProfileWriterBinary::writeHeader(
const StringMap<FunctionSamples> &ProfileMap) {
- writeMagicIdent();
+ writeMagicIdent(Format);
computeSummary(ProfileMap);
if (auto EC = writeSummary())
@@ -248,6 +388,82 @@ std::error_code SampleProfileWriterBinary::writeHeader(
return sampleprof_error::success;
}
+void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
+ for (auto &Entry : SectionHdrLayout)
+ addSecFlags(Entry, SecFlagCompress);
+}
+
+void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
+ addSectionFlags(Type, SecFlagCompress);
+}
+
+void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type,
+ SecFlags Flags) {
+ for (auto &Entry : SectionHdrLayout) {
+ if (Entry.Type == Type)
+ addSecFlags(Entry, Flags);
+ }
+}
+
+void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
+ support::endian::Writer Writer(*OutputStream, support::little);
+
+ Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
+ SecHdrTableOffset = OutputStream->tell();
+ for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
+ Writer.write(static_cast<uint64_t>(-1));
+ Writer.write(static_cast<uint64_t>(-1));
+ Writer.write(static_cast<uint64_t>(-1));
+ Writer.write(static_cast<uint64_t>(-1));
+ }
+}
+
+std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
+ auto &OFS = static_cast<raw_fd_ostream &>(*OutputStream);
+ uint64_t Saved = OutputStream->tell();
+
+ // Set OutputStream to the location saved in SecHdrTableOffset.
+ if (OFS.seek(SecHdrTableOffset) == (uint64_t)-1)
+ return sampleprof_error::ostream_seek_unsupported;
+ support::endian::Writer Writer(*OutputStream, support::little);
+
+ DenseMap<uint32_t, uint32_t> IndexMap;
+ for (uint32_t i = 0; i < SecHdrTable.size(); i++) {
+ IndexMap.insert({static_cast<uint32_t>(SecHdrTable[i].Type), i});
+ }
+
+ // Write the section header table in the order specified in
+ // SectionHdrLayout. That is the sections order Reader will see.
+ // Note that the sections order in which Reader expects to read
+ // may be different from the order in which Writer is able to
+ // write, so we need to adjust the order in SecHdrTable to be
+ // consistent with SectionHdrLayout when we write SecHdrTable
+ // to the memory.
+ for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
+ uint32_t idx = IndexMap[static_cast<uint32_t>(SectionHdrLayout[i].Type)];
+ Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Type));
+ Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Flags));
+ Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Offset));
+ Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Size));
+ }
+
+ // Reset OutputStream.
+ if (OFS.seek(Saved) == (uint64_t)-1)
+ return sampleprof_error::ostream_seek_unsupported;
+
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
+ const StringMap<FunctionSamples> &ProfileMap) {
+ auto &OS = *OutputStream;
+ FileStart = OS.tell();
+ writeMagicIdent(Format);
+
+ allocSecHdrTable();
+ return sampleprof_error::success;
+}
+
std::error_code SampleProfileWriterCompactBinary::writeHeader(
const StringMap<FunctionSamples> &ProfileMap) {
support::endian::Writer Writer(*OutputStream, support::little);
@@ -294,8 +510,8 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
encodeULEB128(Loc.Discriminator, OS);
encodeULEB128(Sample.getSamples(), OS);
encodeULEB128(Sample.getCallTargets().size(), OS);
- for (const auto &J : Sample.getCallTargets()) {
- StringRef Callee = J.first();
+ for (const auto &J : Sample.getSortedCallTargets()) {
+ StringRef Callee = J.first;
uint64_t CalleeSamples = J.second;
if (std::error_code EC = writeNameIdx(Callee))
return EC;
@@ -324,13 +540,14 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
/// Write samples of a top-level function to a binary file.
///
/// \returns true if the samples were written successfully, false otherwise.
-std::error_code SampleProfileWriterBinary::write(const FunctionSamples &S) {
+std::error_code
+SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
encodeULEB128(S.getHeadSamples(), *OutputStream);
return writeBody(S);
}
std::error_code
-SampleProfileWriterCompactBinary::write(const FunctionSamples &S) {
+SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {
uint64_t Offset = OutputStream->tell();
StringRef Name = S.getName();
FuncOffsetTable[Name] = Offset;
@@ -349,10 +566,11 @@ ErrorOr<std::unique_ptr<SampleProfileWriter>>
SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
std::error_code EC;
std::unique_ptr<raw_ostream> OS;
- if (Format == SPF_Binary || Format == SPF_Compact_Binary)
- OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::F_None));
+ if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
+ Format == SPF_Compact_Binary)
+ OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
else
- OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::F_Text));
+ OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_Text));
if (EC)
return EC;
@@ -374,6 +592,8 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
if (Format == SPF_Binary)
Writer.reset(new SampleProfileWriterRawBinary(OS));
+ else if (Format == SPF_Ext_Binary)
+ Writer.reset(new SampleProfileWriterExtBinary(OS));
else if (Format == SPF_Compact_Binary)
Writer.reset(new SampleProfileWriterCompactBinary(OS));
else if (Format == SPF_Text)
@@ -386,6 +606,7 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
if (EC)
return EC;
+ Writer->Format = Format;
return std::move(Writer);
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-30 11:33:16 +0000