diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ProfileData')
11 files changed, 1123 insertions, 626 deletions
diff --git a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index 70f00d333db1..a8cc308b4e3a 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -249,7 +249,12 @@ Error CoverageMapping::loadFunctionRecord( consumeError(std::move(E)); return Error::success(); } - Function.pushRegion(Region, *ExecutionCount); + Expected<int64_t> AltExecutionCount = Ctx.evaluate(Region.FalseCount); + if (auto E = AltExecutionCount.takeError()) { + consumeError(std::move(E)); + return Error::success(); + } + Function.pushRegion(Region, *ExecutionCount, *AltExecutionCount); } // Don't create records for (filenames, function) pairs we've already seen. @@ -485,9 +490,15 @@ class SegmentBuilder { if (CurStartLoc == CR.value().endLoc()) { // Avoid making zero-length regions active. If it's the last region, // emit a skipped segment. Otherwise use its predecessor's count. - const bool Skipped = (CR.index() + 1) == Regions.size(); + const bool Skipped = + (CR.index() + 1) == Regions.size() || + CR.value().Kind == CounterMappingRegion::SkippedRegion; startSegment(ActiveRegions.empty() ? CR.value() : *ActiveRegions.back(), CurStartLoc, !GapRegion, Skipped); + // If it is skipped segment, create a segment with last pushed + // regions's count at CurStartLoc. + if (Skipped && !ActiveRegions.empty()) + startSegment(*ActiveRegions.back(), CurStartLoc, false); continue; } if (CR.index() + 1 == Regions.size() || @@ -587,6 +598,8 @@ public: const auto &L = Segments[I - 1]; const auto &R = Segments[I]; if (!(L.Line < R.Line) && !(L.Line == R.Line && L.Col < R.Col)) { + if (L.Line == R.Line && L.Col == R.Col && !L.HasCount) + continue; LLVM_DEBUG(dbgs() << " ! Segment " << L.Line << ":" << L.Col << " followed by " << R.Line << ":" << R.Col << "\n"); assert(false && "Coverage segments not unique or sorted"); @@ -603,8 +616,7 @@ public: std::vector<StringRef> CoverageMapping::getUniqueSourceFiles() const { std::vector<StringRef> Filenames; for (const auto &Function : getCoveredFunctions()) - Filenames.insert(Filenames.end(), Function.Filenames.begin(), - Function.Filenames.end()); + llvm::append_range(Filenames, Function.Filenames); llvm::sort(Filenames); auto Last = std::unique(Filenames.begin(), Filenames.end()); Filenames.erase(Last, Filenames.end()); @@ -664,6 +676,10 @@ CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const { if (MainFileID && isExpansion(CR, *MainFileID)) FileCoverage.Expansions.emplace_back(CR, Function); } + // Capture branch regions specific to the function (excluding expansions). + for (const auto &CR : Function.CountedBranchRegions) + if (FileIDs.test(CR.FileID) && (CR.FileID == CR.ExpandedFileID)) + FileCoverage.BranchRegions.push_back(CR); } LLVM_DEBUG(dbgs() << "Emitting segments for file: " << Filename << "\n"); @@ -711,6 +727,10 @@ CoverageMapping::getCoverageForFunction(const FunctionRecord &Function) const { if (isExpansion(CR, *MainFileID)) FunctionCoverage.Expansions.emplace_back(CR, Function); } + // Capture branch regions specific to the function (excluding expansions). + for (const auto &CR : Function.CountedBranchRegions) + if (CR.FileID == *MainFileID) + FunctionCoverage.BranchRegions.push_back(CR); LLVM_DEBUG(dbgs() << "Emitting segments for function: " << Function.Name << "\n"); @@ -730,6 +750,10 @@ CoverageData CoverageMapping::getCoverageForExpansion( if (isExpansion(CR, Expansion.FileID)) ExpansionCoverage.Expansions.emplace_back(CR, Expansion.Function); } + for (const auto &CR : Expansion.Function.CountedBranchRegions) + // Capture branch regions that only pertain to the corresponding expansion. + if (CR.FileID == Expansion.FileID) + ExpansionCoverage.BranchRegions.push_back(CR); LLVM_DEBUG(dbgs() << "Emitting segments for expansion of file " << Expansion.FileID << "\n"); @@ -770,6 +794,7 @@ LineCoverageStats::LineCoverageStats( ExecutionCount = WrappedSegment->Count; if (!MinRegionCount) return; + ExecutionCount = 0; for (const auto *LS : LineSegments) if (isStartOfRegion(LS)) ExecutionCount = std::max(ExecutionCount, LS->Count); @@ -807,6 +832,8 @@ static std::string getCoverageMapErrString(coveragemap_error Err) { return "Malformed coverage data"; case coveragemap_error::decompression_failed: return "Failed to decompress coverage data (zlib)"; + case coveragemap_error::invalid_or_missing_arch_specifier: + return "`-arch` specifier is invalid or missing for universal binary"; } llvm_unreachable("A value of coveragemap_error has no message."); } diff --git a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index b75738bc360c..1acdcb4bebb9 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -213,7 +213,7 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray( return Err; unsigned LineStart = 0; for (size_t I = 0; I < NumRegions; ++I) { - Counter C; + Counter C, C2; CounterMappingRegion::RegionKind Kind = CounterMappingRegion::CodeRegion; // Read the combined counter + region kind. @@ -223,6 +223,18 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray( return Err; unsigned Tag = EncodedCounterAndRegion & Counter::EncodingTagMask; uint64_t ExpandedFileID = 0; + + // If Tag does not represent a ZeroCounter, then it is understood to refer + // to a counter or counter expression with region kind assumed to be + // "CodeRegion". In that case, EncodedCounterAndRegion actually encodes the + // referenced counter or counter expression (and nothing else). + // + // If Tag represents a ZeroCounter and EncodingExpansionRegionBit is set, + // then EncodedCounterAndRegion is interpreted to represent an + // ExpansionRegion. In all other cases, EncodedCounterAndRegion is + // interpreted to refer to a specific region kind, after which additional + // fields may be read (e.g. BranchRegions have two encoded counters that + // follow an encoded region kind value). if (Tag != Counter::Zero) { if (auto Err = decodeCounter(EncodedCounterAndRegion, C)) return Err; @@ -243,6 +255,14 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray( case CounterMappingRegion::SkippedRegion: Kind = CounterMappingRegion::SkippedRegion; break; + case CounterMappingRegion::BranchRegion: + // For a Branch Region, read two successive counters. + Kind = CounterMappingRegion::BranchRegion; + if (auto Err = readCounter(C)) + return Err; + if (auto Err = readCounter(C2)) + return Err; + break; default: return make_error<CoverageMapError>(coveragemap_error::malformed); } @@ -294,7 +314,7 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray( dbgs() << "\n"; }); - auto CMR = CounterMappingRegion(C, InferredFileID, ExpandedFileID, + auto CMR = CounterMappingRegion(C, C2, InferredFileID, ExpandedFileID, LineStart, ColumnStart, LineStart + NumLines, ColumnEnd, Kind); if (CMR.startLoc() > CMR.endLoc()) @@ -600,7 +620,7 @@ public: CovBuf += FilenamesSize; FilenameRange FileRange(FilenamesBegin, Filenames.size() - FilenamesBegin); - if (Version == CovMapVersion::Version4) { + if (Version >= CovMapVersion::Version4) { // Map a hash of the filenames region to the filename range associated // with this coverage header. int64_t FilenamesRef = @@ -628,7 +648,7 @@ public: // This is a no-op in Version4 (coverage mappings are not affixed to the // coverage header). const char *MappingBuf = CovBuf; - if (Version == CovMapVersion::Version4 && CoverageSize != 0) + if (Version >= CovMapVersion::Version4 && CoverageSize != 0) return make_error<CoverageMapError>(coveragemap_error::malformed); CovBuf += CoverageSize; const char *MappingEnd = CovBuf; @@ -682,7 +702,7 @@ public: if (FileRange && !FileRange->isInvalid()) { StringRef Mapping = CFR->template getCoverageMapping<Endian>(OutOfLineMappingBuf); - if (Version == CovMapVersion::Version4 && + if (Version >= CovMapVersion::Version4 && Mapping.data() + Mapping.size() > FuncRecBufEnd) return make_error<CoverageMapError>(coveragemap_error::malformed); if (Error Err = insertFunctionRecordIfNeeded(CFR, Mapping, *FileRange)) @@ -711,6 +731,7 @@ Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get( case CovMapVersion::Version2: case CovMapVersion::Version3: case CovMapVersion::Version4: + case CovMapVersion::Version5: // Decompress the name data. if (Error E = P.create(P.getNameData())) return std::move(E); @@ -723,6 +744,9 @@ Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get( else if (Version == CovMapVersion::Version4) return std::make_unique<VersionedCovMapFuncRecordReader< CovMapVersion::Version4, IntPtrT, Endian>>(P, R, F); + else if (Version == CovMapVersion::Version5) + return std::make_unique<VersionedCovMapFuncRecordReader< + CovMapVersion::Version5, IntPtrT, Endian>>(P, R, F); } llvm_unreachable("Unsupported version"); } @@ -766,7 +790,7 @@ static Error readCoverageMappingData( } // In Version4, function records are not affixed to coverage headers. Read // the records from their dedicated section. - if (Version == CovMapVersion::Version4) + if (Version >= CovMapVersion::Version4) return Reader->readFunctionRecords(FuncRecBuf, FuncRecBufEnd, None, nullptr, nullptr); return Error::success(); @@ -950,6 +974,19 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch) { BytesInAddress, Endian); } +/// Determine whether \p Arch is invalid or empty, given \p Bin. +static bool isArchSpecifierInvalidOrMissing(Binary *Bin, StringRef Arch) { + // If we have a universal binary and Arch doesn't identify any of its slices, + // it's user error. + if (auto *Universal = dyn_cast<MachOUniversalBinary>(Bin)) { + for (auto &ObjForArch : Universal->objects()) + if (Arch == ObjForArch.getArchFlagName()) + return false; + return true; + } + return false; +} + Expected<std::vector<std::unique_ptr<BinaryCoverageReader>>> BinaryCoverageReader::create( MemoryBufferRef ObjectBuffer, StringRef Arch, @@ -970,6 +1007,10 @@ BinaryCoverageReader::create( return BinOrErr.takeError(); std::unique_ptr<Binary> Bin = std::move(BinOrErr.get()); + if (isArchSpecifierInvalidOrMissing(Bin.get(), Arch)) + return make_error<CoverageMapError>( + coveragemap_error::invalid_or_missing_arch_specifier); + // MachO universal binaries which contain archives need to be treated as // archives, not as regular binaries. if (auto *Universal = dyn_cast<MachOUniversalBinary>(Bin.get())) { diff --git a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp index 8d3c429c4484..65b83d1f4197 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp @@ -80,10 +80,14 @@ public: ArrayRef<CounterMappingRegion> MappingRegions) : Expressions(Expressions) { AdjustedExpressionIDs.resize(Expressions.size(), 0); - for (const auto &I : MappingRegions) + for (const auto &I : MappingRegions) { mark(I.Count); - for (const auto &I : MappingRegions) + mark(I.FalseCount); + } + for (const auto &I : MappingRegions) { gatherUsed(I.Count); + gatherUsed(I.FalseCount); + } } void mark(Counter C) { @@ -201,6 +205,7 @@ void CoverageMappingWriter::write(raw_ostream &OS) { PrevLineStart = 0; } Counter Count = Minimizer.adjust(I->Count); + Counter FalseCount = Minimizer.adjust(I->FalseCount); switch (I->Kind) { case CounterMappingRegion::CodeRegion: case CounterMappingRegion::GapRegion: @@ -226,6 +231,13 @@ void CoverageMappingWriter::write(raw_ostream &OS) { << Counter::EncodingCounterTagAndExpansionRegionTagBits, OS); break; + case CounterMappingRegion::BranchRegion: + encodeULEB128(unsigned(I->Kind) + << Counter::EncodingCounterTagAndExpansionRegionTagBits, + OS); + writeCounter(MinExpressions, Count, OS); + writeCounter(MinExpressions, FalseCount, OS); + break; } assert(I->LineStart >= PrevLineStart); encodeULEB128(I->LineStart - PrevLineStart, OS); diff --git a/contrib/llvm-project/llvm/lib/ProfileData/GCOV.cpp b/contrib/llvm-project/llvm/lib/ProfileData/GCOV.cpp index 71ea44a1a722..3332a898603b 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/GCOV.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/GCOV.cpp @@ -14,14 +14,16 @@ #include "llvm/ProfileData/GCOV.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" +#include "llvm/Demangle/Demangle.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" -#include "llvm/Support/Path.h" #include "llvm/Support/MD5.h" +#include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <system_error> +#include <unordered_map> using namespace llvm; @@ -39,6 +41,59 @@ enum : uint32_t { GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000, }; +namespace { +struct Summary { + Summary(StringRef Name) : Name(Name) {} + + StringRef Name; + uint64_t lines = 0; + uint64_t linesExec = 0; + uint64_t branches = 0; + uint64_t branchesExec = 0; + uint64_t branchesTaken = 0; +}; + +struct LineInfo { + SmallVector<const GCOVBlock *, 1> blocks; + uint64_t count = 0; + bool exists = false; +}; + +struct SourceInfo { + StringRef filename; + SmallString<0> displayName; + std::vector<std::vector<const GCOVFunction *>> startLineToFunctions; + std::vector<LineInfo> lines; + bool ignored = false; + SourceInfo(StringRef filename) : filename(filename) {} +}; + +class Context { +public: + Context(const GCOV::Options &Options) : options(Options) {} + void print(StringRef filename, StringRef gcno, StringRef gcda, + GCOVFile &file); + +private: + std::string getCoveragePath(StringRef filename, StringRef mainFilename) const; + void printFunctionDetails(const GCOVFunction &f, raw_ostream &os) const; + void printBranchInfo(const GCOVBlock &Block, uint32_t &edgeIdx, + raw_ostream &OS) const; + void printSummary(const Summary &summary, raw_ostream &os) const; + + void collectFunction(GCOVFunction &f, Summary &summary); + void collectSourceLine(SourceInfo &si, Summary *summary, LineInfo &line, + size_t lineNum) const; + void collectSource(SourceInfo &si, Summary &summary) const; + void annotateSource(SourceInfo &si, const GCOVFile &file, StringRef gcno, + StringRef gcda, raw_ostream &os) const; + void printSourceToIntermediate(const SourceInfo &si, raw_ostream &os) const; + + const GCOV::Options &options; + std::vector<SourceInfo> sources; +}; +} // namespace + //===----------------------------------------------------------------------===// // GCOVFile implementation. @@ -56,13 +111,13 @@ bool GCOVFile::readGCNO(GCOVBuffer &buf) { buf.getWord(); // hasUnexecutedBlocks uint32_t tag, length; - GCOVFunction *fn; + GCOVFunction *fn = nullptr; while ((tag = buf.getWord())) { if (!buf.readInt(length)) return false; if (tag == GCOV_TAG_FUNCTION) { - Functions.push_back(std::make_unique<GCOVFunction>(*this)); - fn = Functions.back().get(); + functions.push_back(std::make_unique<GCOVFunction>(*this)); + fn = functions.back().get(); fn->ident = buf.getWord(); fn->linenoChecksum = buf.getWord(); if (Version >= GCOV::V407) @@ -90,41 +145,40 @@ bool GCOVFile::readGCNO(GCOVBuffer &buf) { if (Version < GCOV::V800) { for (uint32_t i = 0; i != length; ++i) { buf.getWord(); // Ignored block flags - fn->Blocks.push_back(std::make_unique<GCOVBlock>(*fn, i)); + fn->blocks.push_back(std::make_unique<GCOVBlock>(i)); } } else { uint32_t num = buf.getWord(); for (uint32_t i = 0; i != num; ++i) - fn->Blocks.push_back(std::make_unique<GCOVBlock>(*fn, i)); + fn->blocks.push_back(std::make_unique<GCOVBlock>(i)); } } else if (tag == GCOV_TAG_ARCS && fn) { uint32_t srcNo = buf.getWord(); - if (srcNo >= fn->Blocks.size()) { + if (srcNo >= fn->blocks.size()) { errs() << "unexpected block number: " << srcNo << " (in " - << fn->Blocks.size() << ")\n"; + << fn->blocks.size() << ")\n"; return false; } - GCOVBlock *src = fn->Blocks[srcNo].get(); + GCOVBlock *src = fn->blocks[srcNo].get(); for (uint32_t i = 0, e = (length - 1) / 2; i != e; ++i) { uint32_t dstNo = buf.getWord(), flags = buf.getWord(); - GCOVBlock *dst = fn->Blocks[dstNo].get(); - auto arc = - std::make_unique<GCOVArc>(*src, *dst, flags & GCOV_ARC_FALLTHROUGH); + GCOVBlock *dst = fn->blocks[dstNo].get(); + auto arc = std::make_unique<GCOVArc>(*src, *dst, flags); src->addDstEdge(arc.get()); dst->addSrcEdge(arc.get()); - if (flags & GCOV_ARC_ON_TREE) + if (arc->onTree()) fn->treeArcs.push_back(std::move(arc)); else fn->arcs.push_back(std::move(arc)); } } else if (tag == GCOV_TAG_LINES && fn) { uint32_t srcNo = buf.getWord(); - if (srcNo >= fn->Blocks.size()) { + if (srcNo >= fn->blocks.size()) { errs() << "unexpected block number: " << srcNo << " (in " - << fn->Blocks.size() << ")\n"; + << fn->blocks.size() << ")\n"; return false; } - GCOVBlock &Block = *fn->Blocks[srcNo]; + GCOVBlock &Block = *fn->blocks[srcNo]; for (;;) { uint32_t line = buf.getWord(); if (line) @@ -219,12 +273,24 @@ bool GCOVFile::readGCDA(GCOVBuffer &buf) { return false; } for (std::unique_ptr<GCOVArc> &arc : fn->arcs) { - if (!buf.readInt64(arc->Count)) + if (!buf.readInt64(arc->count)) return false; - // FIXME Fix counters - arc->src.Counter += arc->Count; - if (arc->dst.succ.empty()) - arc->dst.Counter += arc->Count; + arc->src.count += arc->count; + } + + if (fn->blocks.size() >= 2) { + GCOVBlock &src = *fn->blocks[0]; + GCOVBlock &sink = + Version < GCOV::V408 ? *fn->blocks.back() : *fn->blocks[1]; + auto arc = std::make_unique<GCOVArc>(sink, src, GCOV_ARC_ON_TREE); + sink.addDstEdge(arc.get()); + src.addSrcEdge(arc.get()); + fn->treeArcs.push_back(std::move(arc)); + + for (GCOVBlock &block : fn->blocksRange()) + fn->propagateCounts(block, nullptr); + for (size_t i = fn->treeArcs.size() - 1; i; --i) + fn->treeArcs[i - 1]->src.count += fn->treeArcs[i - 1]->count; } } pos += 4 * length; @@ -246,41 +312,71 @@ void GCOVFile::print(raw_ostream &OS) const { LLVM_DUMP_METHOD void GCOVFile::dump() const { print(dbgs()); } #endif -/// collectLineCounts - Collect line counts. This must be used after -/// reading .gcno and .gcda files. -void GCOVFile::collectLineCounts(FileInfo &fi) { - assert(fi.sources.empty()); - for (StringRef filename : filenames) - fi.sources.emplace_back(filename); - for (GCOVFunction &f : *this) { - f.collectLineCounts(fi); - fi.sources[f.srcIdx].functions.push_back(&f); - } - fi.setRunCount(RunCount); - fi.setProgramCount(ProgramCount); -} +bool GCOVArc::onTree() const { return flags & GCOV_ARC_ON_TREE; } //===----------------------------------------------------------------------===// // GCOVFunction implementation. +StringRef GCOVFunction::getName(bool demangle) const { + if (!demangle) + return Name; + if (demangled.empty()) { + do { + if (Name.startswith("_Z")) { + int status = 0; + // Name is guaranteed to be NUL-terminated. + char *res = itaniumDemangle(Name.data(), nullptr, nullptr, &status); + if (status == 0) { + demangled = res; + free(res); + break; + } + } + demangled = Name; + } while (0); + } + return demangled; +} StringRef GCOVFunction::getFilename() const { return file.filenames[srcIdx]; } /// getEntryCount - Get the number of times the function was called by /// retrieving the entry block's count. uint64_t GCOVFunction::getEntryCount() const { - return Blocks.front()->getCount(); + return blocks.front()->getCount(); } -/// getExitCount - Get the number of times the function returned by retrieving -/// the exit block's count. -uint64_t GCOVFunction::getExitCount() const { - return Blocks.back()->getCount(); +GCOVBlock &GCOVFunction::getExitBlock() const { + return file.getVersion() < GCOV::V408 ? *blocks.back() : *blocks[1]; +} + +// For each basic block, the sum of incoming edge counts equals the sum of +// outgoing edge counts by Kirchoff's circuit law. If the unmeasured arcs form a +// spanning tree, the count for each unmeasured arc (GCOV_ARC_ON_TREE) can be +// uniquely identified. +uint64_t GCOVFunction::propagateCounts(const GCOVBlock &v, GCOVArc *pred) { + // If GCOV_ARC_ON_TREE edges do form a tree, visited is not needed; otherwise + // this prevents infinite recursion. + if (!visited.insert(&v).second) + return 0; + + uint64_t excess = 0; + for (GCOVArc *e : v.srcs()) + if (e != pred) + excess += e->onTree() ? propagateCounts(e->src, e) : e->count; + for (GCOVArc *e : v.dsts()) + if (e != pred) + excess -= e->onTree() ? propagateCounts(e->dst, e) : e->count; + if (int64_t(excess) < 0) + excess = -excess; + if (pred) + pred->count = excess; + return excess; } void GCOVFunction::print(raw_ostream &OS) const { OS << "===== " << Name << " (" << ident << ") @ " << getFilename() << ":" << startLine << "\n"; - for (const auto &Block : Blocks) + for (const auto &Block : blocks) Block->print(OS); } @@ -291,44 +387,30 @@ LLVM_DUMP_METHOD void GCOVFunction::dump() const { print(dbgs()); } /// collectLineCounts - Collect line counts. This must be used after /// reading .gcno and .gcda files. -void GCOVFunction::collectLineCounts(FileInfo &FI) { - // If the line number is zero, this is a function that doesn't actually appear - // in the source file, so there isn't anything we can do with it. - if (startLine == 0) - return; - - for (const auto &Block : Blocks) - Block->collectLineCounts(FI); - FI.addFunctionLine(getFilename(), startLine, this); -} //===----------------------------------------------------------------------===// // GCOVBlock implementation. -/// collectLineCounts - Collect line counts. This must be used after -/// reading .gcno and .gcda files. -void GCOVBlock::collectLineCounts(FileInfo &FI) { - for (uint32_t N : Lines) - FI.addBlockLine(Parent.getFilename(), N, this); -} - void GCOVBlock::print(raw_ostream &OS) const { - OS << "Block : " << Number << " Counter : " << Counter << "\n"; + OS << "Block : " << number << " Counter : " << count << "\n"; if (!pred.empty()) { OS << "\tSource Edges : "; for (const GCOVArc *Edge : pred) - OS << Edge->src.Number << " (" << Edge->Count << "), "; + OS << Edge->src.number << " (" << Edge->count << "), "; OS << "\n"; } if (!succ.empty()) { OS << "\tDestination Edges : "; - for (const GCOVArc *Edge : succ) - OS << Edge->dst.Number << " (" << Edge->Count << "), "; + for (const GCOVArc *Edge : succ) { + if (Edge->flags & GCOV_ARC_ON_TREE) + OS << '*'; + OS << Edge->dst.number << " (" << Edge->count << "), "; + } OS << "\n"; } - if (!Lines.empty()) { + if (!lines.empty()) { OS << "\tLines : "; - for (uint32_t N : Lines) + for (uint32_t N : lines) OS << (N) << ","; OS << "\n"; } @@ -339,139 +421,96 @@ void GCOVBlock::print(raw_ostream &OS) const { LLVM_DUMP_METHOD void GCOVBlock::dump() const { print(dbgs()); } #endif -//===----------------------------------------------------------------------===// -// Cycles detection -// -// The algorithm in GCC is based on the algorithm by Hawick & James: -// "Enumerating Circuits and Loops in Graphs with Self-Arcs and Multiple-Arcs" -// http://complexity.massey.ac.nz/cstn/013/cstn-013.pdf. - -/// Get the count for the detected cycle. -uint64_t GCOVBlock::getCycleCount(const Edges &Path) { - uint64_t CycleCount = std::numeric_limits<uint64_t>::max(); - for (auto E : Path) { - CycleCount = std::min(E->CyclesCount, CycleCount); - } - for (auto E : Path) { - E->CyclesCount -= CycleCount; - } - return CycleCount; -} - -/// Unblock a vertex previously marked as blocked. -void GCOVBlock::unblock(const GCOVBlock *U, BlockVector &Blocked, - BlockVectorLists &BlockLists) { - auto it = find(Blocked, U); - if (it == Blocked.end()) { - return; - } - - const size_t index = it - Blocked.begin(); - Blocked.erase(it); - - const BlockVector ToUnblock(BlockLists[index]); - BlockLists.erase(BlockLists.begin() + index); - for (auto GB : ToUnblock) { - GCOVBlock::unblock(GB, Blocked, BlockLists); - } -} - -bool GCOVBlock::lookForCircuit(const GCOVBlock *V, const GCOVBlock *Start, - Edges &Path, BlockVector &Blocked, - BlockVectorLists &BlockLists, - const BlockVector &Blocks, uint64_t &Count) { - Blocked.push_back(V); - BlockLists.emplace_back(BlockVector()); - bool FoundCircuit = false; - - for (auto E : V->dsts()) { - const GCOVBlock *W = &E->dst; - if (W < Start || find(Blocks, W) == Blocks.end()) { +uint64_t +GCOVBlock::augmentOneCycle(GCOVBlock *src, + std::vector<std::pair<GCOVBlock *, size_t>> &stack) { + GCOVBlock *u; + size_t i; + stack.clear(); + stack.emplace_back(src, 0); + src->incoming = (GCOVArc *)1; // Mark u available for cycle detection + for (;;) { + std::tie(u, i) = stack.back(); + if (i == u->succ.size()) { + u->traversable = false; + stack.pop_back(); + if (stack.empty()) + break; continue; } - - Path.push_back(E); - - if (W == Start) { - // We've a cycle. - Count += GCOVBlock::getCycleCount(Path); - FoundCircuit = true; - } else if (find(Blocked, W) == Blocked.end() && // W is not blocked. - GCOVBlock::lookForCircuit(W, Start, Path, Blocked, BlockLists, - Blocks, Count)) { - FoundCircuit = true; + ++stack.back().second; + GCOVArc *succ = u->succ[i]; + // Ignore saturated arcs (cycleCount has been reduced to 0) and visited + // blocks. Ignore self arcs to guard against bad input (.gcno has no + // self arcs). + if (succ->cycleCount == 0 || !succ->dst.traversable || &succ->dst == u) + continue; + if (succ->dst.incoming == nullptr) { + succ->dst.incoming = succ; + stack.emplace_back(&succ->dst, 0); + continue; } - - Path.pop_back(); - } - - if (FoundCircuit) { - GCOVBlock::unblock(V, Blocked, BlockLists); - } else { - for (auto E : V->dsts()) { - const GCOVBlock *W = &E->dst; - if (W < Start || find(Blocks, W) == Blocks.end()) { - continue; - } - const size_t index = find(Blocked, W) - Blocked.begin(); - BlockVector &List = BlockLists[index]; - if (find(List, V) == List.end()) { - List.push_back(V); - } + uint64_t minCount = succ->cycleCount; + for (GCOVBlock *v = u;;) { + minCount = std::min(minCount, v->incoming->cycleCount); + v = &v->incoming->src; + if (v == &succ->dst) + break; } + succ->cycleCount -= minCount; + for (GCOVBlock *v = u;;) { + v->incoming->cycleCount -= minCount; + v = &v->incoming->src; + if (v == &succ->dst) + break; + } + return minCount; } - - return FoundCircuit; -} - -/// Get the count for the list of blocks which lie on the same line. -void GCOVBlock::getCyclesCount(const BlockVector &Blocks, uint64_t &Count) { - for (auto Block : Blocks) { - Edges Path; - BlockVector Blocked; - BlockVectorLists BlockLists; - - GCOVBlock::lookForCircuit(Block, Block, Path, Blocked, BlockLists, Blocks, - Count); - } + return 0; } -/// Get the count for the list of blocks which lie on the same line. -uint64_t GCOVBlock::getLineCount(const BlockVector &Blocks) { - uint64_t Count = 0; - - for (auto Block : Blocks) { - if (Block->getNumSrcEdges() == 0) { - // The block has no predecessors and a non-null counter - // (can be the case with entry block in functions). - Count += Block->getCount(); - } else { - // Add counts from predecessors that are not on the same line. - for (auto E : Block->srcs()) { - const GCOVBlock *W = &E->src; - if (find(Blocks, W) == Blocks.end()) { - Count += E->Count; - } - } +// Get the total execution count of loops among blocks on the same line. +// Assuming a reducible flow graph, the count is the sum of back edge counts. +// Identifying loops is complex, so we simply find cycles and perform cycle +// cancelling iteratively. +uint64_t GCOVBlock::getCyclesCount(const BlockVector &blocks) { + std::vector<std::pair<GCOVBlock *, size_t>> stack; + uint64_t count = 0, d; + for (;;) { + // Make blocks on the line traversable and try finding a cycle. + for (auto b : blocks) { + const_cast<GCOVBlock *>(b)->traversable = true; + const_cast<GCOVBlock *>(b)->incoming = nullptr; } - for (auto E : Block->dsts()) { - E->CyclesCount = E->Count; + d = 0; + for (auto block : blocks) { + auto *b = const_cast<GCOVBlock *>(block); + if (b->traversable && (d = augmentOneCycle(b, stack)) > 0) + break; } + if (d == 0) + break; + count += d; } - - GCOVBlock::getCyclesCount(Blocks, Count); - - return Count; + // If there is no more loop, all traversable bits should have been cleared. + // This property is needed by subsequent calls. + for (auto b : blocks) { + assert(!b->traversable); + (void)b; + } + return count; } //===----------------------------------------------------------------------===// // FileInfo implementation. -// Safe integer division, returns 0 if numerator is 0. -static uint32_t safeDiv(uint64_t Numerator, uint64_t Divisor) { - if (!Numerator) +// Format dividend/divisor as a percentage. Return 1 if the result is greater +// than 0% and less than 1%. +static uint32_t formatPercentage(uint64_t dividend, uint64_t divisor) { + if (!dividend || !divisor) return 0; - return Numerator / Divisor; + dividend *= 100; + return dividend < divisor ? 1 : dividend / divisor; } // This custom division function mimics gcov's branch ouputs: @@ -522,8 +561,11 @@ class LineConsumer { public: LineConsumer() = default; LineConsumer(StringRef Filename) { + // Open source files without requiring a NUL terminator. The concurrent + // modification may nullify the NUL terminator condition. ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = - MemoryBuffer::getFileOrSTDIN(Filename); + MemoryBuffer::getFileOrSTDIN(Filename, -1, + /*RequiresNullTerminator=*/false); if (std::error_code EC = BufferOrErr.getError()) { errs() << Filename << ": " << EC.message() << "\n"; Remaining = ""; @@ -579,23 +621,23 @@ static std::string mangleCoveragePath(StringRef Filename, bool PreservePaths) { return std::string(Result.str()); } -std::string FileInfo::getCoveragePath(StringRef Filename, - StringRef MainFilename) { - if (Options.NoOutput) +std::string Context::getCoveragePath(StringRef filename, + StringRef mainFilename) const { + if (options.NoOutput) // This is probably a bug in gcov, but when -n is specified, paths aren't // mangled at all, and the -l and -p options are ignored. Here, we do the // same. - return std::string(Filename); + return std::string(filename); std::string CoveragePath; - if (Options.LongFileNames && !Filename.equals(MainFilename)) + if (options.LongFileNames && !filename.equals(mainFilename)) CoveragePath = - mangleCoveragePath(MainFilename, Options.PreservePaths) + "##"; - CoveragePath += mangleCoveragePath(Filename, Options.PreservePaths); - if (Options.HashFilenames) { + mangleCoveragePath(mainFilename, options.PreservePaths) + "##"; + CoveragePath += mangleCoveragePath(filename, options.PreservePaths); + if (options.HashFilenames) { MD5 Hasher; MD5::MD5Result Result; - Hasher.update(Filename.str()); + Hasher.update(filename.str()); Hasher.final(Result); CoveragePath += "##" + std::string(Result.digest()); } @@ -603,292 +645,302 @@ std::string FileInfo::getCoveragePath(StringRef Filename, return CoveragePath; } -std::unique_ptr<raw_ostream> -FileInfo::openCoveragePath(StringRef CoveragePath) { - std::error_code EC; - auto OS = - std::make_unique<raw_fd_ostream>(CoveragePath, EC, sys::fs::OF_Text); - if (EC) { - errs() << EC.message() << "\n"; - return std::make_unique<raw_null_ostream>(); +void Context::collectFunction(GCOVFunction &f, Summary &summary) { + SourceInfo &si = sources[f.srcIdx]; + if (f.startLine >= si.startLineToFunctions.size()) + si.startLineToFunctions.resize(f.startLine + 1); + si.startLineToFunctions[f.startLine].push_back(&f); + for (const GCOVBlock &b : f.blocksRange()) { + if (b.lines.empty()) + continue; + uint32_t maxLineNum = *std::max_element(b.lines.begin(), b.lines.end()); + if (maxLineNum >= si.lines.size()) + si.lines.resize(maxLineNum + 1); + for (uint32_t lineNum : b.lines) { + LineInfo &line = si.lines[lineNum]; + if (!line.exists) + ++summary.lines; + if (line.count == 0 && b.count) + ++summary.linesExec; + line.exists = true; + line.count += b.count; + line.blocks.push_back(&b); + } } - return std::move(OS); } -/// print - Print source files with collected line count information. -void FileInfo::print(raw_ostream &InfoOS, StringRef MainFilename, - StringRef GCNOFile, StringRef GCDAFile, GCOVFile &file) { - SmallVector<StringRef, 4> Filenames; - for (const auto &LI : LineInfo) - Filenames.push_back(LI.first()); - llvm::sort(Filenames); - - for (StringRef Filename : Filenames) { - auto AllLines = - Options.Intermediate ? LineConsumer() : LineConsumer(Filename); - std::string CoveragePath = getCoveragePath(Filename, MainFilename); - std::unique_ptr<raw_ostream> CovStream; - if (Options.NoOutput || Options.Intermediate) - CovStream = std::make_unique<raw_null_ostream>(); - else if (!Options.UseStdout) - CovStream = openCoveragePath(CoveragePath); - raw_ostream &CovOS = - !Options.NoOutput && Options.UseStdout ? llvm::outs() : *CovStream; - - CovOS << " -: 0:Source:" << Filename << "\n"; - CovOS << " -: 0:Graph:" << GCNOFile << "\n"; - CovOS << " -: 0:Data:" << GCDAFile << "\n"; - CovOS << " -: 0:Runs:" << RunCount << "\n"; - if (file.getVersion() < GCOV::V900) - CovOS << " -: 0:Programs:" << ProgramCount << "\n"; - - const LineData &Line = LineInfo[Filename]; - GCOVCoverage FileCoverage(Filename); - for (uint32_t LineIndex = 0; LineIndex < Line.LastLine || !AllLines.empty(); - ++LineIndex) { - if (Options.BranchInfo) { - FunctionLines::const_iterator FuncsIt = Line.Functions.find(LineIndex); - if (FuncsIt != Line.Functions.end()) - printFunctionSummary(CovOS, FuncsIt->second); - } +void Context::collectSourceLine(SourceInfo &si, Summary *summary, + LineInfo &line, size_t lineNum) const { + uint64_t count = 0; + for (const GCOVBlock *b : line.blocks) { + if (b->number == 0) { + // For nonstandard control flows, arcs into the exit block may be + // duplicately counted (fork) or not be counted (abnormal exit), and thus + // the (exit,entry) counter may be inaccurate. Count the entry block with + // the outgoing arcs. + for (const GCOVArc *arc : b->succ) + count += arc->count; + } else { + // Add counts from predecessors that are not on the same line. + for (const GCOVArc *arc : b->pred) + if (!llvm::is_contained(line.blocks, &arc->src)) + count += arc->count; + } + for (GCOVArc *arc : b->succ) + arc->cycleCount = arc->count; + } - BlockLines::const_iterator BlocksIt = Line.Blocks.find(LineIndex); - if (BlocksIt == Line.Blocks.end()) { - // No basic blocks are on this line. Not an executable line of code. - CovOS << " -:"; - AllLines.printNext(CovOS, LineIndex + 1); - } else { - const BlockVector &Blocks = BlocksIt->second; - - // Add up the block counts to form line counts. - DenseMap<const GCOVFunction *, bool> LineExecs; - for (const GCOVBlock *Block : Blocks) { - if (Options.FuncCoverage) { - // This is a slightly convoluted way to most accurately gather line - // statistics for functions. Basically what is happening is that we - // don't want to count a single line with multiple blocks more than - // once. However, we also don't simply want to give the total line - // count to every function that starts on the line. Thus, what is - // happening here are two things: - // 1) Ensure that the number of logical lines is only incremented - // once per function. - // 2) If there are multiple blocks on the same line, ensure that the - // number of lines executed is incremented as long as at least - // one of the blocks are executed. - const GCOVFunction *Function = &Block->getParent(); - if (FuncCoverages.find(Function) == FuncCoverages.end()) { - std::pair<const GCOVFunction *, GCOVCoverage> KeyValue( - Function, GCOVCoverage(Function->getName())); - FuncCoverages.insert(KeyValue); - } - GCOVCoverage &FuncCoverage = FuncCoverages.find(Function)->second; - - if (LineExecs.find(Function) == LineExecs.end()) { - if (Block->getCount()) { - ++FuncCoverage.LinesExec; - LineExecs[Function] = true; - } else { - LineExecs[Function] = false; - } - ++FuncCoverage.LogicalLines; - } else if (!LineExecs[Function] && Block->getCount()) { - ++FuncCoverage.LinesExec; - LineExecs[Function] = true; - } - } - } + count += GCOVBlock::getCyclesCount(line.blocks); + line.count = count; + if (line.exists) { + ++summary->lines; + if (line.count != 0) + ++summary->linesExec; + } - const uint64_t LineCount = GCOVBlock::getLineCount(Blocks); - if (LineCount == 0) - CovOS << " #####:"; - else { - CovOS << format("%9" PRIu64 ":", LineCount); - ++FileCoverage.LinesExec; - } - ++FileCoverage.LogicalLines; - - AllLines.printNext(CovOS, LineIndex + 1); - - uint32_t BlockNo = 0; - uint32_t EdgeNo = 0; - for (const GCOVBlock *Block : Blocks) { - // Only print block and branch information at the end of the block. - if (Block->getLastLine() != LineIndex + 1) - continue; - if (Options.AllBlocks) - printBlockInfo(CovOS, *Block, LineIndex, BlockNo); - if (Options.BranchInfo) { - size_t NumEdges = Block->getNumDstEdges(); - if (NumEdges > 1) - printBranchInfo(CovOS, *Block, FileCoverage, EdgeNo); - else if (Options.UncondBranch && NumEdges == 1) - printUncondBranchInfo(CovOS, EdgeNo, Block->succ[0]->Count); - } - } + if (options.BranchInfo) + for (const GCOVBlock *b : line.blocks) { + if (b->getLastLine() != lineNum) + continue; + int branches = 0, execBranches = 0, takenBranches = 0; + for (const GCOVArc *arc : b->succ) { + ++branches; + if (count != 0) + ++execBranches; + if (arc->count != 0) + ++takenBranches; + } + if (branches > 1) { + summary->branches += branches; + summary->branchesExec += execBranches; + summary->branchesTaken += takenBranches; } } - SourceInfo &source = sources[file.filenameToIdx.find(Filename)->second]; - source.name = CoveragePath; - source.coverage = FileCoverage; +} + +void Context::collectSource(SourceInfo &si, Summary &summary) const { + size_t lineNum = 0; + for (LineInfo &line : si.lines) { + collectSourceLine(si, &summary, line, lineNum); + ++lineNum; } +} - if (Options.Intermediate && !Options.NoOutput) { - // gcov 7.* unexpectedly create multiple .gcov files, which was fixed in 8.0 - // (PR GCC/82702). We create just one file. - std::string outputPath(sys::path::filename(MainFilename)); - std::error_code ec; - raw_fd_ostream os(outputPath + ".gcov", ec, sys::fs::OF_Text); - if (ec) { - errs() << ec.message() << "\n"; - return; +void Context::annotateSource(SourceInfo &si, const GCOVFile &file, + StringRef gcno, StringRef gcda, + raw_ostream &os) const { + auto source = + options.Intermediate ? LineConsumer() : LineConsumer(si.filename); + + os << " -: 0:Source:" << si.displayName << '\n'; + os << " -: 0:Graph:" << gcno << '\n'; + os << " -: 0:Data:" << gcda << '\n'; + os << " -: 0:Runs:" << file.RunCount << '\n'; + if (file.Version < GCOV::V900) + os << " -: 0:Programs:" << file.ProgramCount << '\n'; + + for (size_t lineNum = 1; !source.empty(); ++lineNum) { + if (lineNum >= si.lines.size()) { + os << " -:"; + source.printNext(os, lineNum); + continue; } - for (const SourceInfo &source : sources) { - os << "file:" << source.filename << '\n'; - for (const GCOVFunction *f : source.functions) - os << "function:" << f->startLine << ',' << f->getEntryCount() << ',' - << f->Name << '\n'; - const LineData &line = LineInfo[source.filename]; - for (uint32_t lineNum = 0; lineNum != line.LastLine; ++lineNum) { - BlockLines::const_iterator BlocksIt = line.Blocks.find(lineNum); - if (BlocksIt == line.Blocks.end()) - continue; - const BlockVector &blocks = BlocksIt->second; - // GCC 8 (r254259) added third third field for Ada: - // lcount:<line>,<count>,<has_unexecuted_blocks> - // We don't need the third field. - os << "lcount:" << (lineNum + 1) << ',' - << GCOVBlock::getLineCount(blocks) << '\n'; - - if (!Options.BranchInfo) - continue; - for (const GCOVBlock *block : blocks) { - if (block->getLastLine() != lineNum + 1 || - block->getNumDstEdges() < 2) - continue; - for (const GCOVArc *arc : block->dsts()) { - const char *type = block->getCount() - ? arc->Count ? "taken" : "nottaken" - : "notexec"; - os << "branch:" << (lineNum + 1) << ',' << type << '\n'; - } + const LineInfo &line = si.lines[lineNum]; + if (options.BranchInfo && lineNum < si.startLineToFunctions.size()) + for (const auto *f : si.startLineToFunctions[lineNum]) + printFunctionDetails(*f, os); + if (!line.exists) + os << " -:"; + else if (line.count == 0) + os << " #####:"; + else + os << format("%9" PRIu64 ":", line.count); + source.printNext(os, lineNum); + + uint32_t blockIdx = 0, edgeIdx = 0; + for (const GCOVBlock *b : line.blocks) { + if (b->getLastLine() != lineNum) + continue; + if (options.AllBlocks) { + if (b->getCount() == 0) + os << " $$$$$:"; + else + os << format("%9" PRIu64 ":", b->count); + os << format("%5u-block %2u\n", lineNum, blockIdx++); + } + if (options.BranchInfo) { + size_t NumEdges = b->succ.size(); + if (NumEdges > 1) + printBranchInfo(*b, edgeIdx, os); + else if (options.UncondBranch && NumEdges == 1) { + uint64_t count = b->succ[0]->count; + os << format("unconditional %2u ", edgeIdx++) + << formatBranchInfo(options, count, count) << '\n'; } } } } +} + +void Context::printSourceToIntermediate(const SourceInfo &si, + raw_ostream &os) const { + os << "file:" << si.filename << '\n'; + for (const auto &fs : si.startLineToFunctions) + for (const GCOVFunction *f : fs) + os << "function:" << f->startLine << ',' << f->getEntryCount() << ',' + << f->getName(options.Demangle) << '\n'; + for (size_t lineNum = 1, size = si.lines.size(); lineNum < size; ++lineNum) { + const LineInfo &line = si.lines[lineNum]; + if (line.blocks.empty()) + continue; + // GCC 8 (r254259) added third third field for Ada: + // lcount:<line>,<count>,<has_unexecuted_blocks> + // We don't need the third field. + os << "lcount:" << lineNum << ',' << line.count << '\n'; - if (!Options.UseStdout) { - // FIXME: There is no way to detect calls given current instrumentation. - if (Options.FuncCoverage) - printFuncCoverage(InfoOS); - printFileCoverage(InfoOS); + if (!options.BranchInfo) + continue; + for (const GCOVBlock *b : line.blocks) { + if (b->succ.size() < 2 || b->getLastLine() != lineNum) + continue; + for (const GCOVArc *arc : b->succ) { + const char *type = + b->getCount() ? arc->count ? "taken" : "nottaken" : "notexec"; + os << "branch:" << lineNum << ',' << type << '\n'; + } + } } } -/// printFunctionSummary - Print function and block summary. -void FileInfo::printFunctionSummary(raw_ostream &OS, - const FunctionVector &Funcs) const { - for (const GCOVFunction *Func : Funcs) { - uint64_t EntryCount = Func->getEntryCount(); - uint32_t BlocksExec = 0; - for (const GCOVBlock &Block : Func->blocks()) - if (Block.getNumDstEdges() && Block.getCount()) - ++BlocksExec; - - OS << "function " << Func->getName() << " called " << EntryCount - << " returned " << safeDiv(Func->getExitCount() * 100, EntryCount) - << "% blocks executed " - << safeDiv(BlocksExec * 100, Func->getNumBlocks() - 1) << "%\n"; +void Context::print(StringRef filename, StringRef gcno, StringRef gcda, + GCOVFile &file) { + for (StringRef filename : file.filenames) { + sources.emplace_back(filename); + SourceInfo &si = sources.back(); + si.displayName = si.filename; + if (!options.SourcePrefix.empty() && + sys::path::replace_path_prefix(si.displayName, options.SourcePrefix, + "") && + !si.displayName.empty()) { + // TODO replace_path_prefix may strip the prefix even if the remaining + // part does not start with a separator. + if (sys::path::is_separator(si.displayName[0])) + si.displayName.erase(si.displayName.begin()); + else + si.displayName = si.filename; + } + if (options.RelativeOnly && sys::path::is_absolute(si.displayName)) + si.ignored = true; } -} -/// printBlockInfo - Output counts for each block. -void FileInfo::printBlockInfo(raw_ostream &OS, const GCOVBlock &Block, - uint32_t LineIndex, uint32_t &BlockNo) const { - if (Block.getCount() == 0) - OS << " $$$$$:"; - else - OS << format("%9" PRIu64 ":", Block.getCount()); - OS << format("%5u-block %2u\n", LineIndex + 1, BlockNo++); -} + raw_ostream &os = llvm::outs(); + for (GCOVFunction &f : make_pointee_range(file.functions)) { + Summary summary(f.getName(options.Demangle)); + collectFunction(f, summary); + if (options.FuncCoverage && !options.UseStdout) { + os << "Function '" << summary.Name << "'\n"; + printSummary(summary, os); + os << '\n'; + } + } -/// printBranchInfo - Print conditional branch probabilities. -void FileInfo::printBranchInfo(raw_ostream &OS, const GCOVBlock &Block, - GCOVCoverage &Coverage, uint32_t &EdgeNo) { - SmallVector<uint64_t, 16> BranchCounts; - uint64_t TotalCounts = 0; - for (const GCOVArc *Edge : Block.dsts()) { - BranchCounts.push_back(Edge->Count); - TotalCounts += Edge->Count; - if (Block.getCount()) - ++Coverage.BranchesExec; - if (Edge->Count) - ++Coverage.BranchesTaken; - ++Coverage.Branches; - - if (Options.FuncCoverage) { - const GCOVFunction *Function = &Block.getParent(); - GCOVCoverage &FuncCoverage = FuncCoverages.find(Function)->second; - if (Block.getCount()) - ++FuncCoverage.BranchesExec; - if (Edge->Count) - ++FuncCoverage.BranchesTaken; - ++FuncCoverage.Branches; + for (SourceInfo &si : sources) { + if (si.ignored) + continue; + Summary summary(si.displayName); + collectSource(si, summary); + + // Print file summary unless -t is specified. + std::string gcovName = getCoveragePath(si.filename, filename); + if (!options.UseStdout) { + os << "File '" << summary.Name << "'\n"; + printSummary(summary, os); + if (!options.NoOutput && !options.Intermediate) + os << "Creating '" << gcovName << "'\n"; + os << '\n'; + } + + if (options.NoOutput || options.Intermediate) + continue; + Optional<raw_fd_ostream> os; + if (!options.UseStdout) { + std::error_code ec; + os.emplace(gcovName, ec, sys::fs::OF_Text); + if (ec) { + errs() << ec.message() << '\n'; + continue; + } + } + annotateSource(si, file, gcno, gcda, + options.UseStdout ? llvm::outs() : *os); + } + + if (options.Intermediate && !options.NoOutput) { + // gcov 7.* unexpectedly create multiple .gcov files, which was fixed in 8.0 + // (PR GCC/82702). We create just one file. + std::string outputPath(sys::path::filename(filename)); + std::error_code ec; + raw_fd_ostream os(outputPath + ".gcov", ec, sys::fs::OF_Text); + if (ec) { + errs() << ec.message() << '\n'; + return; } + + for (const SourceInfo &si : sources) + printSourceToIntermediate(si, os); } +} - for (uint64_t N : BranchCounts) - OS << format("branch %2u ", EdgeNo++) - << formatBranchInfo(Options, N, TotalCounts) << "\n"; +void Context::printFunctionDetails(const GCOVFunction &f, + raw_ostream &os) const { + const uint64_t entryCount = f.getEntryCount(); + uint32_t blocksExec = 0; + const GCOVBlock &exitBlock = f.getExitBlock(); + uint64_t exitCount = 0; + for (const GCOVArc *arc : exitBlock.pred) + exitCount += arc->count; + for (const GCOVBlock &b : f.blocksRange()) + if (b.number != 0 && &b != &exitBlock && b.getCount()) + ++blocksExec; + + os << "function " << f.getName(options.Demangle) << " called " << entryCount + << " returned " << formatPercentage(exitCount, entryCount) + << "% blocks executed " + << formatPercentage(blocksExec, f.blocks.size() - 2) << "%\n"; } -/// printUncondBranchInfo - Print unconditional branch probabilities. -void FileInfo::printUncondBranchInfo(raw_ostream &OS, uint32_t &EdgeNo, - uint64_t Count) const { - OS << format("unconditional %2u ", EdgeNo++) - << formatBranchInfo(Options, Count, Count) << "\n"; +/// printBranchInfo - Print conditional branch probabilities. +void Context::printBranchInfo(const GCOVBlock &Block, uint32_t &edgeIdx, + raw_ostream &os) const { + uint64_t total = 0; + for (const GCOVArc *arc : Block.dsts()) + total += arc->count; + for (const GCOVArc *arc : Block.dsts()) + os << format("branch %2u ", edgeIdx++) + << formatBranchInfo(options, arc->count, total) << '\n'; } -// printCoverage - Print generic coverage info used by both printFuncCoverage -// and printFileCoverage. -void FileInfo::printCoverage(raw_ostream &OS, - const GCOVCoverage &Coverage) const { - OS << format("Lines executed:%.2f%% of %u\n", - double(Coverage.LinesExec) * 100 / Coverage.LogicalLines, - Coverage.LogicalLines); - if (Options.BranchInfo) { - if (Coverage.Branches) { - OS << format("Branches executed:%.2f%% of %u\n", - double(Coverage.BranchesExec) * 100 / Coverage.Branches, - Coverage.Branches); - OS << format("Taken at least once:%.2f%% of %u\n", - double(Coverage.BranchesTaken) * 100 / Coverage.Branches, - Coverage.Branches); +void Context::printSummary(const Summary &summary, raw_ostream &os) const { + os << format("Lines executed:%.2f%% of %" PRIu64 "\n", + double(summary.linesExec) * 100 / summary.lines, summary.lines); + if (options.BranchInfo) { + if (summary.branches == 0) { + os << "No branches\n"; } else { - OS << "No branches\n"; + os << format("Branches executed:%.2f%% of %" PRIu64 "\n", + double(summary.branchesExec) * 100 / summary.branches, + summary.branches); + os << format("Taken at least once:%.2f%% of %" PRIu64 "\n", + double(summary.branchesTaken) * 100 / summary.branches, + summary.branches); } - OS << "No calls\n"; // to be consistent with gcov - } -} - -// printFuncCoverage - Print per-function coverage info. -void FileInfo::printFuncCoverage(raw_ostream &OS) const { - for (const auto &FC : FuncCoverages) { - const GCOVCoverage &Coverage = FC.second; - OS << "Function '" << Coverage.Name << "'\n"; - printCoverage(OS, Coverage); - OS << "\n"; + os << "No calls\n"; } } -// printFileCoverage - Print per-file coverage info. -void FileInfo::printFileCoverage(raw_ostream &OS) const { - for (const SourceInfo &source : sources) { - const GCOVCoverage &Coverage = source.coverage; - OS << "File '" << Coverage.Name << "'\n"; - printCoverage(OS, Coverage); - if (!Options.NoOutput && !Options.Intermediate) - OS << "Creating '" << source.name << "'\n"; - OS << "\n"; - } +void llvm::gcovOneInput(const GCOV::Options &options, StringRef filename, + StringRef gcno, StringRef gcda, GCOVFile &file) { + Context fi(options); + fi.print(filename, gcno, gcda, file); } diff --git a/contrib/llvm-project/llvm/lib/ProfileData/InstrProf.cpp b/contrib/llvm-project/llvm/lib/ProfileData/InstrProf.cpp index b9d8ae9ba60d..4a0cee67089d 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/InstrProf.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/InstrProf.cpp @@ -625,11 +625,11 @@ void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, } } -void InstrProfValueSiteRecord::scale(uint64_t Weight, +void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn) { for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) { bool Overflowed; - I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed); + I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D; if (Overflowed) Warn(instrprof_error::counter_overflow); } @@ -678,22 +678,23 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, } void InstrProfRecord::scaleValueProfData( - uint32_t ValueKind, uint64_t Weight, + uint32_t ValueKind, uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn) { for (auto &R : getValueSitesForKind(ValueKind)) - R.scale(Weight, Warn); + R.scale(N, D, Warn); } -void InstrProfRecord::scale(uint64_t Weight, +void InstrProfRecord::scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn) { + assert(D != 0 && "D cannot be 0"); for (auto &Count : this->Counts) { bool Overflowed; - Count = SaturatingMultiply(Count, Weight, &Overflowed); + Count = SaturatingMultiply(Count, N, &Overflowed) / D; if (Overflowed) Warn(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - scaleValueProfData(Kind, Weight, Warn); + scaleValueProfData(Kind, N, D, Warn); } // Map indirect call target name hash to name string. @@ -1111,35 +1112,17 @@ bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { return true; } -// Parse the value profile options. -void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart, - int64_t &RangeLast) { - static const int64_t DefaultMemOPSizeRangeStart = 0; - static const int64_t DefaultMemOPSizeRangeLast = 8; - RangeStart = DefaultMemOPSizeRangeStart; - RangeLast = DefaultMemOPSizeRangeLast; - - if (!MemOPSizeRange.empty()) { - auto Pos = MemOPSizeRange.find(':'); - if (Pos != std::string::npos) { - if (Pos > 0) - MemOPSizeRange.substr(0, Pos).getAsInteger(10, RangeStart); - if (Pos < MemOPSizeRange.size() - 1) - MemOPSizeRange.substr(Pos + 1).getAsInteger(10, RangeLast); - } else - MemOPSizeRange.getAsInteger(10, RangeLast); - } - assert(RangeLast >= RangeStart); -} - // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime // aware this is an ir_level profile so it can set the version flag. -void createIRLevelProfileFlagVar(Module &M, bool IsCS) { +void createIRLevelProfileFlagVar(Module &M, bool IsCS, + bool InstrEntryBBEnabled) { const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); Type *IntTy64 = Type::getInt64Ty(M.getContext()); uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); if (IsCS) ProfileVersion |= VARIANT_MASK_CSIR_PROF; + if (InstrEntryBBEnabled) + ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; auto IRLevelVersionVariable = new GlobalVariable( M, IntTy64, true, GlobalValue::WeakAnyLinkage, Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName); diff --git a/contrib/llvm-project/llvm/lib/ProfileData/InstrProfReader.cpp b/contrib/llvm-project/llvm/lib/ProfileData/InstrProfReader.cpp index 16a69cb5457b..9581e5b486a6 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/InstrProfReader.cpp @@ -154,23 +154,29 @@ bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) { Error TextInstrProfReader::readHeader() { Symtab.reset(new InstrProfSymtab()); bool IsIRInstr = false; - if (!Line->startswith(":")) { - IsIRLevelProfile = false; - return success(); + bool IsEntryFirst = false; + bool IsCS = false; + + while (Line->startswith(":")) { + StringRef Str = Line->substr(1); + if (Str.equals_lower("ir")) + IsIRInstr = true; + else if (Str.equals_lower("fe")) + IsIRInstr = false; + else if (Str.equals_lower("csir")) { + IsIRInstr = true; + IsCS = true; + } else if (Str.equals_lower("entry_first")) + IsEntryFirst = true; + else if (Str.equals_lower("not_entry_first")) + IsEntryFirst = false; + else + return error(instrprof_error::bad_header); + ++Line; } - StringRef Str = (Line)->substr(1); - if (Str.equals_lower("ir")) - IsIRInstr = true; - else if (Str.equals_lower("fe")) - IsIRInstr = false; - else if (Str.equals_lower("csir")) { - IsIRInstr = true; - HasCSIRLevelProfile = true; - } else - return error(instrprof_error::bad_header); - - ++Line; IsIRLevelProfile = IsIRInstr; + InstrEntryBBEnabled = IsEntryFirst; + HasCSIRLevelProfile = IsCS; return success(); } diff --git a/contrib/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp b/contrib/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp index ccb270e0b719..d07668322354 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -165,8 +165,9 @@ public: } // end namespace llvm -InstrProfWriter::InstrProfWriter(bool Sparse) - : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {} +InstrProfWriter::InstrProfWriter(bool Sparse, bool InstrEntryBBEnabled) + : Sparse(Sparse), InstrEntryBBEnabled(InstrEntryBBEnabled), + InfoObj(new InstrProfRecordWriterTrait()) {} InstrProfWriter::~InstrProfWriter() { delete InfoObj; } @@ -240,7 +241,7 @@ void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash, // We've never seen a function with this name and hash, add it. Dest = std::move(I); if (Weight > 1) - Dest.scale(Weight, MapWarn); + Dest.scale(Weight, 1, MapWarn); } else { // We're updating a function we've seen before. Dest.merge(I, Weight, MapWarn); @@ -308,6 +309,9 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) { Header.Version |= VARIANT_MASK_IR_PROF; Header.Version |= VARIANT_MASK_CSIR_PROF; } + if (InstrEntryBBEnabled) + Header.Version |= VARIANT_MASK_INSTR_ENTRY; + Header.Unused = 0; Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType); Header.HashOffset = 0; @@ -441,6 +445,8 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { OS << "# IR level Instrumentation Flag\n:ir\n"; else if (ProfileKind == PF_IRLevelWithCS) OS << "# CSIR level Instrumentation Flag\n:csir\n"; + if (InstrEntryBBEnabled) + OS << "# Always instrument the function entry block\n:entry_first\n"; InstrProfSymtab Symtab; using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>; diff --git a/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp index 5d3a07640942..d2603097c550 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -119,13 +119,22 @@ std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() { } void InstrProfSummaryBuilder::addEntryCount(uint64_t Count) { - addCount(Count); NumFunctions++; + + // Skip invalid count. + if (Count == (uint64_t)-1) + return; + + addCount(Count); if (Count > MaxFunctionCount) MaxFunctionCount = Count; } void InstrProfSummaryBuilder::addInternalCount(uint64_t Count) { + // Skip invalid count. + if (Count == (uint64_t)-1) + return; + addCount(Count); if (Count > MaxInternalBlockCount) MaxInternalBlockCount = Count; diff --git a/contrib/llvm-project/llvm/lib/ProfileData/SampleProf.cpp b/contrib/llvm-project/llvm/lib/ProfileData/SampleProf.cpp index e5d0fdba5fc4..d6acc00e1a6f 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/SampleProf.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/SampleProf.cpp @@ -14,6 +14,8 @@ #include "llvm/ProfileData/SampleProf.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/PseudoProbe.h" +#include "llvm/ProfileData/SampleProfReader.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" @@ -30,6 +32,8 @@ using namespace sampleprof; namespace llvm { namespace sampleprof { SampleProfileFormat FunctionSamples::Format; +bool FunctionSamples::ProfileIsProbeBased = false; +bool FunctionSamples::ProfileIsCS = false; bool FunctionSamples::UseMD5; } // namespace sampleprof } // namespace llvm @@ -75,6 +79,8 @@ class SampleProfErrorCategoryType : public std::error_category { return "Uncompress failure"; case sampleprof_error::zlib_unavailable: return "Zlib is unavailable"; + case sampleprof_error::hash_mismatch: + return "Function hash mismatch"; } llvm_unreachable("A value of sampleprof_error has no message."); } @@ -127,6 +133,9 @@ raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS, /// Print the samples collected for a function on stream \p OS. void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { + if (getFunctionHash()) + OS << "CFG checksum " << getFunctionHash() << "\n"; + OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size() << " sampled lines\n"; @@ -174,8 +183,22 @@ unsigned FunctionSamples::getOffset(const DILocation *DIL) { 0xffff; } -const FunctionSamples * -FunctionSamples::findFunctionSamples(const DILocation *DIL) const { +LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) { + if (FunctionSamples::ProfileIsProbeBased) + // In a pseudo-probe based profile, a callsite is simply represented by the + // ID of the probe associated with the call instruction. The probe ID is + // encoded in the Discriminator field of the call instruction's debug + // metadata. + return LineLocation(PseudoProbeDwarfDiscriminator::extractProbeIndex( + DIL->getDiscriminator()), + 0); + else + return LineLocation(FunctionSamples::getOffset(DIL), + DIL->getBaseDiscriminator()); +} + +const FunctionSamples *FunctionSamples::findFunctionSamples( + const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const { assert(DIL); SmallVector<std::pair<LineLocation, StringRef>, 10> S; @@ -190,11 +213,59 @@ FunctionSamples::findFunctionSamples(const DILocation *DIL) const { return this; const FunctionSamples *FS = this; for (int i = S.size() - 1; i >= 0 && FS != nullptr; i--) { - FS = FS->findFunctionSamplesAt(S[i].first, S[i].second); + FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper); } return FS; } +void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const { + NameSet.insert(Name); + for (const auto &BS : BodySamples) + for (const auto &TS : BS.second.getCallTargets()) + NameSet.insert(TS.getKey()); + + for (const auto &CS : CallsiteSamples) { + for (const auto &NameFS : CS.second) { + NameSet.insert(NameFS.first); + NameFS.second.findAllNames(NameSet); + } + } +} + +const FunctionSamples *FunctionSamples::findFunctionSamplesAt( + const LineLocation &Loc, StringRef CalleeName, + SampleProfileReaderItaniumRemapper *Remapper) const { + std::string CalleeGUID; + CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID); + + auto iter = CallsiteSamples.find(Loc); + if (iter == CallsiteSamples.end()) + return nullptr; + auto FS = iter->second.find(CalleeName); + if (FS != iter->second.end()) + return &FS->second; + if (Remapper) { + if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) { + auto FS = iter->second.find(*NameInProfile); + if (FS != iter->second.end()) + return &FS->second; + } + } + // If we cannot find exact match of the callee name, return the FS with + // the max total count. Only do this when CalleeName is not provided, + // i.e., only for indirect calls. + if (!CalleeName.empty()) + return nullptr; + uint64_t MaxTotalSamples = 0; + const FunctionSamples *R = nullptr; + for (const auto &NameFS : iter->second) + if (NameFS.second.getTotalSamples() >= MaxTotalSamples) { + MaxTotalSamples = NameFS.second.getTotalSamples(); + R = &NameFS.second; + } + return R; +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void FunctionSamples::dump() const { print(dbgs(), 0); } #endif @@ -216,8 +287,7 @@ std::error_code ProfileSymbolList::read(const uint8_t *Data, std::error_code ProfileSymbolList::write(raw_ostream &OS) { // Sort the symbols before output. If doing compression. // It will make the compression much more effective. - std::vector<StringRef> SortedList; - SortedList.insert(SortedList.begin(), Syms.begin(), Syms.end()); + std::vector<StringRef> SortedList(Syms.begin(), Syms.end()); llvm::sort(SortedList); std::string OutputString; @@ -232,8 +302,7 @@ std::error_code ProfileSymbolList::write(raw_ostream &OS) { void ProfileSymbolList::dump(raw_ostream &OS) const { OS << "======== Dump profile symbol list ========\n"; - std::vector<StringRef> SortedList; - SortedList.insert(SortedList.begin(), Syms.begin(), Syms.end()); + std::vector<StringRef> SortedList(Syms.begin(), Syms.end()); llvm::sort(SortedList); for (auto &Sym : SortedList) diff --git a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp index 03f1ac190b91..c42931174bc0 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfReader.cpp @@ -83,26 +83,52 @@ static bool ParseHead(const StringRef &Input, StringRef &FName, /// Returns true if line offset \p L is legal (only has 16 bits). static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } +/// Parse \p Input that contains metadata. +/// Possible metadata: +/// - CFG Checksum information: +/// !CFGChecksum: 12345 +/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash. +static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash) { + if (!Input.startswith("!CFGChecksum:")) + return false; + + StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim(); + return !CFGInfo.getAsInteger(10, FunctionHash); +} + +enum class LineType { + CallSiteProfile, + BodyProfile, + Metadata, +}; + /// Parse \p Input as line sample. /// /// \param Input input line. -/// \param IsCallsite true if the line represents an inlined callsite. +/// \param LineTy Type of this line. /// \param Depth the depth of the inline stack. /// \param NumSamples total samples of the line/inlined callsite. /// \param LineOffset line offset to the start of the function. /// \param Discriminator discriminator of the line. /// \param TargetCountMap map from indirect call target to count. +/// \param FunctionHash the function's CFG hash, used by pseudo probe. /// /// returns true if parsing is successful. -static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, +static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, - DenseMap<StringRef, uint64_t> &TargetCountMap) { + DenseMap<StringRef, uint64_t> &TargetCountMap, + uint64_t &FunctionHash) { for (Depth = 0; Input[Depth] == ' '; Depth++) ; if (Depth == 0) return false; + if (Depth == 1 && Input[Depth] == '!') { + LineTy = LineType::Metadata; + return parseMetadata(Input.substr(Depth), FunctionHash); + } + size_t n1 = Input.find(':'); StringRef Loc = Input.substr(Depth, n1 - Depth); size_t n2 = Loc.find('.'); @@ -118,8 +144,8 @@ static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, } StringRef Rest = Input.substr(n1 + 2); - if (Rest[0] >= '0' && Rest[0] <= '9') { - IsCallsite = false; + if (isDigit(Rest[0])) { + LineTy = LineType::BodyProfile; size_t n3 = Rest.find(' '); if (n3 == StringRef::npos) { if (Rest.getAsInteger(10, NumSamples)) @@ -176,7 +202,7 @@ static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, n3 = n4; } } else { - IsCallsite = true; + LineTy = LineType::CallSiteProfile; size_t n3 = Rest.find_last_of(':'); CalleeName = Rest.substr(0, n3); if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples)) @@ -196,6 +222,13 @@ std::error_code SampleProfileReaderText::readImpl() { sampleprof_error Result = sampleprof_error::success; InlineCallStack InlineStack; + int CSProfileCount = 0; + int RegularProfileCount = 0; + uint32_t ProbeProfileCount = 0; + + // SeenMetadata tracks whether we have processed metadata for the current + // top-level function profile. + bool SeenMetadata = false; for (; !LineIt.is_at_eof(); ++LineIt) { if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') @@ -220,9 +253,16 @@ std::error_code SampleProfileReaderText::readImpl() { "Expected 'mangled_name:NUM:NUM', found " + *LineIt); return sampleprof_error::malformed; } - Profiles[FName] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[FName]; - FProfile.setName(FName); + SeenMetadata = false; + SampleContext FContext(FName); + if (FContext.hasContext()) + ++CSProfileCount; + else + ++RegularProfileCount; + Profiles[FContext] = FunctionSamples(); + FunctionSamples &FProfile = Profiles[FContext]; + FProfile.setName(FContext.getName()); + FProfile.setContext(FContext); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); InlineStack.clear(); @@ -231,25 +271,35 @@ std::error_code SampleProfileReaderText::readImpl() { uint64_t NumSamples; StringRef FName; DenseMap<StringRef, uint64_t> TargetCountMap; - bool IsCallsite; uint32_t Depth, LineOffset, Discriminator; - if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset, - Discriminator, FName, TargetCountMap)) { + LineType LineTy; + uint64_t FunctionHash; + if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset, + Discriminator, FName, TargetCountMap, FunctionHash)) { reportError(LineIt.line_number(), "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); return sampleprof_error::malformed; } - if (IsCallsite) { - while (InlineStack.size() > Depth) { - InlineStack.pop_back(); - } + if (SeenMetadata && LineTy != LineType::Metadata) { + // Metadata must be put at the end of a function profile. + reportError(LineIt.line_number(), + "Found non-metadata after metadata: " + *LineIt); + return sampleprof_error::malformed; + } + while (InlineStack.size() > Depth) { + InlineStack.pop_back(); + } + switch (LineTy) { + case LineType::CallSiteProfile: { FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( LineLocation(LineOffset, Discriminator))[std::string(FName)]; FSamples.setName(FName); MergeResult(Result, FSamples.addTotalSamples(NumSamples)); InlineStack.push_back(&FSamples); - } else { + break; + } + case LineType::BodyProfile: { while (InlineStack.size() > Depth) { InlineStack.pop_back(); } @@ -261,9 +311,27 @@ std::error_code SampleProfileReaderText::readImpl() { } MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, NumSamples)); + break; + } + case LineType::Metadata: { + FunctionSamples &FProfile = *InlineStack.back(); + FProfile.setFunctionHash(FunctionHash); + ++ProbeProfileCount; + SeenMetadata = true; + break; + } } } } + + assert((RegularProfileCount == 0 || CSProfileCount == 0) && + "Cannot have both context-sensitive and regular profile"); + ProfileIsCS = (CSProfileCount > 0); + assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) && + "Cannot have both probe-based profiles and regular profiles"); + ProfileIsProbeBased = (ProbeProfileCount > 0); + FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + if (Result == sampleprof_error::success) computeSummary(); @@ -354,6 +422,34 @@ ErrorOr<StringRef> SampleProfileReaderBinary::readStringFromTable() { return NameTable[*Idx]; } +ErrorOr<StringRef> SampleProfileReaderExtBinaryBase::readStringFromTable() { + if (!FixedLengthMD5) + return SampleProfileReaderBinary::readStringFromTable(); + + // read NameTable index. + auto Idx = readStringIndex(NameTable); + if (std::error_code EC = Idx.getError()) + return EC; + + // Check whether the name to be accessed has been accessed before, + // if not, read it from memory directly. + StringRef &SR = NameTable[*Idx]; + if (SR.empty()) { + const uint8_t *SavedData = Data; + Data = MD5NameMemStart + ((*Idx) * sizeof(uint64_t)); + auto FID = readUnencodedNumber<uint64_t>(); + if (std::error_code EC = FID.getError()) + return EC; + // Save the string converted from uint64_t in MD5StringBuf. All the + // references to the name are all StringRefs refering to the string + // in MD5StringBuf. + MD5StringBuf->push_back(std::to_string(*FID)); + SR = MD5StringBuf->back(); + Data = SavedData; + } + return SR; +} + ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() { auto Idx = readStringIndex(NameTable); if (std::error_code EC = Idx.getError()) @@ -470,7 +566,7 @@ std::error_code SampleProfileReaderBinary::readImpl() { return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinary::readOneSection( +std::error_code SampleProfileReaderExtBinaryBase::readOneSection( const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) { Data = Start; End = Start + Size; @@ -481,37 +577,56 @@ std::error_code SampleProfileReaderExtBinary::readOneSection( if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) Summary->setPartialProfile(true); break; - case SecNameTable: - if (std::error_code EC = readNameTableSec( - hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))) + case SecNameTable: { + FixedLengthMD5 = + hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5); + bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name); + assert((!FixedLengthMD5 || UseMD5) && + "If FixedLengthMD5 is true, UseMD5 has to be true"); + if (std::error_code EC = readNameTableSec(UseMD5)) return EC; break; + } case SecLBRProfile: if (std::error_code EC = readFuncProfiles()) return EC; break; - case SecProfileSymbolList: - if (std::error_code EC = readProfileSymbolList()) - return EC; - break; case SecFuncOffsetTable: if (std::error_code EC = readFuncOffsetTable()) return EC; break; + case SecFuncMetadata: + ProfileIsProbeBased = + hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); + FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + if (std::error_code EC = readFuncMetadata()) + return EC; + break; + case SecProfileSymbolList: + if (std::error_code EC = readProfileSymbolList()) + return EC; + break; default: + if (std::error_code EC = readCustomSection(Entry)) + return EC; break; } return sampleprof_error::success; } -void SampleProfileReaderExtBinary::collectFuncsFrom(const Module &M) { +void SampleProfileReaderExtBinaryBase::collectFuncsFrom(const Module &M) { UseAllFuncs = false; FuncsToUse.clear(); for (auto &F : M) FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); } -std::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() { +std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { + // If there are more than one FuncOffsetTable, the profile read associated + // with previous FuncOffsetTable has to be done before next FuncOffsetTable + // is read. + FuncOffsetTable.clear(); + auto Size = readNumber<uint64_t>(); if (std::error_code EC = Size.getError()) return EC; @@ -531,7 +646,7 @@ std::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() { return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinary::readFuncProfiles() { +std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { const uint8_t *Start = Data; if (UseAllFuncs) { while (Data < End) { @@ -576,7 +691,7 @@ std::error_code SampleProfileReaderExtBinary::readFuncProfiles() { return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinary::readProfileSymbolList() { +std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() { if (!ProfSymList) ProfSymList = std::make_unique<ProfileSymbolList>(); @@ -625,6 +740,10 @@ std::error_code SampleProfileReaderExtBinaryBase::readImpl() { if (!Entry.Size) continue; + // Skip sections without context when SkipFlatProf is true. + if (SkipFlatProf && hasSecFlag(Entry, SecCommonFlags::SecFlagFlat)) + continue; + const uint8_t *SecStart = BufStart + Entry.Offset; uint64_t SecSize = Entry.Size; @@ -709,7 +828,7 @@ std::error_code SampleProfileReaderBinary::readNameTable() { auto Size = readNumber<uint32_t>(); if (std::error_code EC = Size.getError()) return EC; - NameTable.reserve(*Size); + NameTable.reserve(*Size + NameTable.size()); for (uint32_t I = 0; I < *Size; ++I) { auto Name(readString()); if (std::error_code EC = Name.getError()) @@ -720,13 +839,24 @@ std::error_code SampleProfileReaderBinary::readNameTable() { return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinary::readMD5NameTable() { +std::error_code SampleProfileReaderExtBinaryBase::readMD5NameTable() { auto Size = readNumber<uint64_t>(); if (std::error_code EC = Size.getError()) return EC; - NameTable.reserve(*Size); MD5StringBuf = std::make_unique<std::vector<std::string>>(); MD5StringBuf->reserve(*Size); + if (FixedLengthMD5) { + // Preallocate and initialize NameTable so we can check whether a name + // index has been read before by checking whether the element in the + // NameTable is empty, meanwhile readStringIndex can do the boundary + // check using the size of NameTable. + NameTable.resize(*Size + NameTable.size()); + + MD5NameMemStart = Data; + Data = Data + (*Size) * sizeof(uint64_t); + return sampleprof_error::success; + } + NameTable.reserve(*Size); for (uint32_t I = 0; I < *Size; ++I) { auto FID = readNumber<uint64_t>(); if (std::error_code EC = FID.getError()) @@ -739,12 +869,29 @@ std::error_code SampleProfileReaderExtBinary::readMD5NameTable() { return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinary::readNameTableSec(bool IsMD5) { +std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5) { if (IsMD5) return readMD5NameTable(); return SampleProfileReaderBinary::readNameTable(); } +std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() { + if (!ProfileIsProbeBased) + return sampleprof_error::success; + for (unsigned I = 0; I < Profiles.size(); ++I) { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + + auto Checksum = readNumber<uint64_t>(); + if (std::error_code EC = Checksum.getError()) + return EC; + + Profiles[*FName].setFunctionHash(*Checksum); + } + return sampleprof_error::success; +} + std::error_code SampleProfileReaderCompactBinary::readNameTable() { auto Size = readNumber<uint64_t>(); if (std::error_code EC = Size.getError()) @@ -759,7 +906,8 @@ std::error_code SampleProfileReaderCompactBinary::readNameTable() { return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTableEntry() { +std::error_code +SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint32_t Idx) { SecHdrTableEntry Entry; auto Type = readUnencodedNumber<uint64_t>(); if (std::error_code EC = Type.getError()) @@ -781,6 +929,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTableEntry() { return EC; Entry.Size = *Size; + Entry.LayoutIndex = Idx; SecHdrTable.push_back(std::move(Entry)); return sampleprof_error::success; } @@ -791,7 +940,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() { return EC; for (uint32_t i = 0; i < (*EntryNum); i++) - if (std::error_code EC = readSecHdrTableEntry()) + if (std::error_code EC = readSecHdrTableEntry(i)) return EC; return sampleprof_error::success; @@ -813,11 +962,12 @@ std::error_code SampleProfileReaderExtBinaryBase::readHeader() { } uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) { + uint64_t Size = 0; for (auto &Entry : SecHdrTable) { if (Entry.Type == Type) - return Entry.Size; + Size += Entry.Size; } - return 0; + return Size; } uint64_t SampleProfileReaderExtBinaryBase::getFileSize() { @@ -840,9 +990,14 @@ static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) { else Flags.append("{"); + if (hasSecFlag(Entry, SecCommonFlags::SecFlagFlat)) + Flags.append("flat,"); + switch (Entry.Type) { case SecNameTable: - if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)) + if (hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5)) + Flags.append("fixlenmd5,"); + else if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)) Flags.append("md5,"); break; case SecProfSummary: @@ -867,7 +1022,7 @@ bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry) << "\n"; ; - TotalSecsSize += getSectionSize(Entry.Type); + TotalSecsSize += Entry.Size; } uint64_t HeaderSize = SecHdrTable.front().Offset; assert(HeaderSize + TotalSecsSize == getFileSize() && @@ -1201,7 +1356,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( InlineCallStack NewStack; NewStack.push_back(FProfile); - NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); + llvm::append_range(NewStack, InlineStack); if (Update) { // Walk up the inline stack, adding the samples on this line to // the total sample count of the callers in the chain. @@ -1249,7 +1404,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( return sampleprof_error::truncated; InlineCallStack NewStack; NewStack.push_back(FProfile); - NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); + llvm::append_range(NewStack, InlineStack); if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset)) return EC; } @@ -1290,19 +1445,25 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { return; } + // CSSPGO-TODO: Remapper is not yet supported. + // We will need to remap the entire context string. assert(Remappings && "should be initialized while creating remapper"); - for (auto &Sample : Reader.getProfiles()) - if (auto Key = Remappings->insert(Sample.first())) - SampleMap.insert({Key, &Sample.second}); + for (auto &Sample : Reader.getProfiles()) { + DenseSet<StringRef> NamesInSample; + Sample.second.findAllNames(NamesInSample); + for (auto &Name : NamesInSample) + if (auto Key = Remappings->insert(Name)) + NameMap.insert({Key, Name}); + } RemappingApplied = true; } -FunctionSamples * -SampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) { +Optional<StringRef> +SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) { if (auto Key = Remappings->lookup(Fname)) - return SampleMap.lookup(Key); - return nullptr; + return NameMap.lookup(Key); + return None; } /// Prepare a memory buffer for the contents of \p Filename. diff --git a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp index 48d3faa6cd2f..71dba6281f76 100644 --- a/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -19,6 +19,7 @@ #include "llvm/ProfileData/SampleProfWriter.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Compression.h" @@ -73,19 +74,16 @@ SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) { return sampleprof_error::success; } -SecHdrTableEntry & -SampleProfileWriterExtBinaryBase::getEntryInLayout(SecType Type) { - auto SecIt = std::find_if( - SectionHdrLayout.begin(), SectionHdrLayout.end(), - [=](const auto &Entry) -> bool { return Entry.Type == Type; }); - return *SecIt; -} - /// Return the current position and prepare to use it as the start -/// position of a section. -uint64_t SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type) { +/// position of a section given the section type \p Type and its position +/// \p LayoutIdx in SectionHdrLayout. +uint64_t +SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type, + uint32_t LayoutIdx) { uint64_t SectionStart = OutputStream->tell(); - auto &Entry = getEntryInLayout(Type); + assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range"); + const auto &Entry = SectionHdrLayout[LayoutIdx]; + assert(Entry.Type == Type && "Unexpected section type"); // Use LocalBuf as a temporary output for writting data. if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) LocalBufStream.swap(OutputStream); @@ -112,18 +110,21 @@ std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() { return sampleprof_error::success; } -/// Add a new section into section header table. -std::error_code -SampleProfileWriterExtBinaryBase::addNewSection(SecType Type, - uint64_t SectionStart) { - auto Entry = getEntryInLayout(Type); +/// Add a new section into section header table given the section type +/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the +/// location \p SectionStart where the section should be written to. +std::error_code SampleProfileWriterExtBinaryBase::addNewSection( + SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) { + assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range"); + const auto &Entry = SectionHdrLayout[LayoutIdx]; + assert(Entry.Type == Type && "Unexpected section type"); if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) { LocalBufStream.swap(OutputStream); if (std::error_code EC = compressAndOutput()) return EC; } SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart, - OutputStream->tell() - SectionStart}); + OutputStream->tell() - SectionStart, LayoutIdx}); return sampleprof_error::success; } @@ -144,7 +145,7 @@ std::error_code SampleProfileWriterExtBinaryBase::write( } std::error_code -SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) { +SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { uint64_t Offset = OutputStream->tell(); StringRef Name = S.getName(); FuncOffsetTable[Name] = Offset - SecLBRProfileStart; @@ -152,7 +153,7 @@ SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) { return writeBody(S); } -std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() { +std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() { auto &OS = *OutputStream; // Write out the table size. @@ -163,10 +164,23 @@ std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() { writeNameIdx(entry.first); encodeULEB128(entry.second, OS); } + FuncOffsetTable.clear(); return sampleprof_error::success; } -std::error_code SampleProfileWriterExtBinary::writeNameTable() { +std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( + const StringMap<FunctionSamples> &Profiles) { + if (!FunctionSamples::ProfileIsProbeBased) + return sampleprof_error::success; + auto &OS = *OutputStream; + for (const auto &Entry : Profiles) { + writeNameIdx(Entry.first()); + encodeULEB128(Entry.second.getFunctionHash(), OS); + } + return sampleprof_error::success; +} + +std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { if (!UseMD5) return SampleProfileWriterBinary::writeNameTable(); @@ -174,59 +188,159 @@ std::error_code SampleProfileWriterExtBinary::writeNameTable() { std::set<StringRef> V; stablizeNameTable(V); - // Write out the name table. + // Write out the MD5 name table. We wrote unencoded MD5 so reader can + // retrieve the name using the name index without having to read the + // whole name table. encodeULEB128(NameTable.size(), OS); - for (auto N : V) { - encodeULEB128(MD5Hash(N), OS); - } + support::endian::Writer Writer(OS, support::little); + for (auto N : V) + Writer.write(MD5Hash(N)); return sampleprof_error::success; } -std::error_code SampleProfileWriterExtBinary::writeSections( +std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( const StringMap<FunctionSamples> &ProfileMap) { - uint64_t SectionStart = markSectionStart(SecProfSummary); - computeSummary(ProfileMap); - if (auto EC = writeSummary()) - return EC; - if (std::error_code EC = addNewSection(SecProfSummary, SectionStart)) - return EC; - - // Generate the name table for all the functions referenced in the profile. - SectionStart = markSectionStart(SecNameTable); for (const auto &I : ProfileMap) { addName(I.first()); addNames(I.second); } - writeNameTable(); - if (std::error_code EC = addNewSection(SecNameTable, SectionStart)) + if (auto EC = writeNameTable()) return EC; + return sampleprof_error::success; +} - SectionStart = markSectionStart(SecLBRProfile); - SecLBRProfileStart = OutputStream->tell(); - if (std::error_code EC = writeFuncProfiles(ProfileMap)) - return EC; - if (std::error_code EC = addNewSection(SecLBRProfile, SectionStart)) - return EC; - - if (ProfSymList && ProfSymList->toCompress()) - setToCompressSection(SecProfileSymbolList); - - SectionStart = markSectionStart(SecProfileSymbolList); +std::error_code +SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() { if (ProfSymList && ProfSymList->size() > 0) if (std::error_code EC = ProfSymList->write(*OutputStream)) return EC; - if (std::error_code EC = addNewSection(SecProfileSymbolList, SectionStart)) + + return sampleprof_error::success; +} + +std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( + SecType Type, uint32_t LayoutIdx, + const StringMap<FunctionSamples> &ProfileMap) { + // The setting of SecFlagCompress should happen before markSectionStart. + if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress()) + setToCompressSection(SecProfileSymbolList); + if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased) + addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased); + + uint64_t SectionStart = markSectionStart(Type, LayoutIdx); + switch (Type) { + case SecProfSummary: + computeSummary(ProfileMap); + if (auto EC = writeSummary()) + return EC; + break; + case SecNameTable: + if (auto EC = writeNameTableSection(ProfileMap)) + return EC; + break; + case SecLBRProfile: + SecLBRProfileStart = OutputStream->tell(); + if (std::error_code EC = writeFuncProfiles(ProfileMap)) + return EC; + break; + case SecFuncOffsetTable: + if (auto EC = writeFuncOffsetTable()) + return EC; + break; + case SecFuncMetadata: + if (std::error_code EC = writeFuncMetadata(ProfileMap)) + return EC; + break; + case SecProfileSymbolList: + if (auto EC = writeProfileSymbolListSection()) + return EC; + break; + default: + if (auto EC = writeCustomSection(Type)) + return EC; + break; + } + if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart)) return EC; + return sampleprof_error::success; +} - SectionStart = markSectionStart(SecFuncOffsetTable); - if (std::error_code EC = writeFuncOffsetTable()) +std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( + const StringMap<FunctionSamples> &ProfileMap) { + // The const indices passed to writeOneSection below are specifying the + // positions of the sections in SectionHdrLayout. Look at + // initSectionHdrLayout to find out where each section is located in + // SectionHdrLayout. + if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecLBRProfile, 3, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecProfileSymbolList, 4, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecFuncMetadata, 5, ProfileMap)) + return EC; + return sampleprof_error::success; +} + +static void +splitProfileMapToTwo(const StringMap<FunctionSamples> &ProfileMap, + StringMap<FunctionSamples> &ContextProfileMap, + StringMap<FunctionSamples> &NoContextProfileMap) { + for (const auto &I : ProfileMap) { + if (I.second.getCallsiteSamples().size()) + ContextProfileMap.insert({I.first(), I.second}); + else + NoContextProfileMap.insert({I.first(), I.second}); + } +} + +std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout( + const StringMap<FunctionSamples> &ProfileMap) { + StringMap<FunctionSamples> ContextProfileMap, NoContextProfileMap; + splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap); + + if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap)) return EC; - if (std::error_code EC = addNewSection(SecFuncOffsetTable, SectionStart)) + if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap)) + return EC; + // Mark the section to have no context. Note section flag needs to be set + // before writing the section. + addSectionFlag(5, SecCommonFlags::SecFlagFlat); + if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap)) + return EC; + // Mark the section to have no context. Note section flag needs to be set + // before writing the section. + addSectionFlag(4, SecCommonFlags::SecFlagFlat); + if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap)) + return EC; + if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap)) return EC; return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinary::writeSections( + const StringMap<FunctionSamples> &ProfileMap) { + std::error_code EC; + if (SecLayout == DefaultLayout) + EC = writeDefaultLayout(ProfileMap); + else if (SecLayout == CtxSplitLayout) + EC = writeCtxSplitLayout(ProfileMap); + else + llvm_unreachable("Unsupported layout"); + return EC; +} + std::error_code SampleProfileWriterCompactBinary::write( const StringMap<FunctionSamples> &ProfileMap) { if (std::error_code EC = SampleProfileWriter::write(ProfileMap)) @@ -246,7 +360,10 @@ std::error_code SampleProfileWriterCompactBinary::write( /// it needs to be parsed by the SampleProfileReaderText class. std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { auto &OS = *OutputStream; - OS << S.getName() << ":" << S.getTotalSamples(); + if (FunctionSamples::ProfileIsCS) + OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples(); + else + OS << S.getName() << ":" << S.getTotalSamples(); if (Indent == 0) OS << ":" << S.getHeadSamples(); OS << "\n"; @@ -285,6 +402,13 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { } Indent -= 1; + if (Indent == 0) { + if (FunctionSamples::ProfileIsProbeBased) { + OS.indent(Indent + 1); + OS << "!CFGChecksum: " << S.getFunctionHash() << "\n"; + } + } + return sampleprof_error::success; } @@ -435,24 +559,31 @@ std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() { return sampleprof_error::ostream_seek_unsupported; support::endian::Writer Writer(*OutputStream, support::little); - DenseMap<uint32_t, uint32_t> IndexMap; - for (uint32_t i = 0; i < SecHdrTable.size(); i++) { - IndexMap.insert({static_cast<uint32_t>(SecHdrTable[i].Type), i}); + assert(SecHdrTable.size() == SectionHdrLayout.size() && + "SecHdrTable entries doesn't match SectionHdrLayout"); + SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1); + for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) { + IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx; } // Write the section header table in the order specified in - // SectionHdrLayout. That is the sections order Reader will see. - // Note that the sections order in which Reader expects to read - // may be different from the order in which Writer is able to - // write, so we need to adjust the order in SecHdrTable to be - // consistent with SectionHdrLayout when we write SecHdrTable - // to the memory. - for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) { - uint32_t idx = IndexMap[static_cast<uint32_t>(SectionHdrLayout[i].Type)]; - Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Type)); - Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Flags)); - Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Offset)); - Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Size)); + // SectionHdrLayout. SectionHdrLayout specifies the sections + // order in which profile reader expect to read, so the section + // header table should be written in the order in SectionHdrLayout. + // Note that the section order in SecHdrTable may be different + // from the order in SectionHdrLayout, for example, SecFuncOffsetTable + // needs to be computed after SecLBRProfile (the order in SecHdrTable), + // but it needs to be read before SecLBRProfile (the order in + // SectionHdrLayout). So we use IndexMap above to switch the order. + for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size(); + LayoutIdx++) { + assert(IndexMap[LayoutIdx] < SecHdrTable.size() && + "Incorrect LayoutIdx in SecHdrTable"); + auto Entry = SecHdrTable[IndexMap[LayoutIdx]]; + Writer.write(static_cast<uint64_t>(Entry.Type)); + Writer.write(static_cast<uint64_t>(Entry.Flags)); + Writer.write(static_cast<uint64_t>(Entry.Offset)); + Writer.write(static_cast<uint64_t>(Entry.Size)); } // Reset OutputStream. |