aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/ProfileData/InstrProf.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/ProfileData/InstrProf.cpp')
-rw-r--r--llvm/lib/ProfileData/InstrProf.cpp334
1 files changed, 225 insertions, 109 deletions
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 0f9c33de3f52..236b083a1e21 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -27,6 +27,7 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Mangler.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
@@ -135,6 +136,9 @@ static std::string getInstrProfErrString(instrprof_error Err,
case instrprof_error::count_mismatch:
OS << "function basic block count change detected (counter mismatch)";
break;
+ case instrprof_error::bitmap_mismatch:
+ OS << "function bitmap size change detected (bitmap size mismatch)";
+ break;
case instrprof_error::counter_overflow:
OS << "counter overflow";
break;
@@ -157,6 +161,9 @@ static std::string getInstrProfErrString(instrprof_error Err,
case instrprof_error::raw_profile_version_mismatch:
OS << "raw profile version mismatch";
break;
+ case instrprof_error::counter_value_too_large:
+ OS << "excessively large counter value suggests corrupted profile data";
+ break;
}
// If optional error message is not empty, append it to the message.
@@ -264,11 +271,56 @@ static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
return PathNameStr.substr(LastPos);
}
-// Return the PGOFuncName. This function has some special handling when called
-// in LTO optimization. The following only applies when calling in LTO passes
-// (when \c InLTO is true): LTO's internalization privatizes many global linkage
-// symbols. This happens after value profile annotation, but those internal
-// linkage functions should not have a source prefix.
+static StringRef getStrippedSourceFileName(const GlobalObject &GO) {
+ StringRef FileName(GO.getParent()->getSourceFileName());
+ uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
+ if (StripLevel < StaticFuncStripDirNamePrefix)
+ StripLevel = StaticFuncStripDirNamePrefix;
+ if (StripLevel)
+ FileName = stripDirPrefix(FileName, StripLevel);
+ return FileName;
+}
+
+// The PGO name has the format [<filepath>;]<linkage-name> where <filepath>; is
+// provided if linkage is local and <linkage-name> is the mangled function
+// name. The filepath is used to discriminate possibly identical function names.
+// ; is used because it is unlikely to be found in either <filepath> or
+// <linkage-name>.
+//
+// Older compilers used getPGOFuncName() which has the format
+// [<filepath>:]<function-name>. <filepath> is used to discriminate between
+// possibly identical function names when linkage is local and <function-name>
+// simply comes from F.getName(). This caused trouble for Objective-C functions
+// which commonly have :'s in their names. Also, since <function-name> is not
+// mangled, they cannot be passed to Mach-O linkers via -order_file. We still
+// need to compute this name to lookup functions from profiles built by older
+// compilers.
+static std::string
+getIRPGONameForGlobalObject(const GlobalObject &GO,
+ GlobalValue::LinkageTypes Linkage,
+ StringRef FileName) {
+ SmallString<64> Name;
+ if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
+ Name.append(FileName.empty() ? "<unknown>" : FileName);
+ Name.append(";");
+ }
+ Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true);
+ return Name.str().str();
+}
+
+static std::optional<std::string> lookupPGONameFromMetadata(MDNode *MD) {
+ if (MD != nullptr) {
+ StringRef S = cast<MDString>(MD->getOperand(0))->getString();
+ return S.str();
+ }
+ return {};
+}
+
+// Returns the PGO object name. This function has some special handling
+// when called in LTO optimization. The following only applies when calling in
+// LTO passes (when \c InLTO is true): LTO's internalization privatizes many
+// global linkage symbols. This happens after value profile annotation, but
+// those internal linkage functions should not have a source prefix.
// Additionally, for ThinLTO mode, exported internal functions are promoted
// and renamed. We need to ensure that the original internal PGO name is
// used when computing the GUID that is compared against the profiled GUIDs.
@@ -277,22 +329,42 @@ static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
// symbols in the value profile annotation step
// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
// data, its original linkage must be non-internal.
+static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
+ MDNode *PGONameMetadata) {
+ if (!InLTO) {
+ auto FileName = getStrippedSourceFileName(GO);
+ return getIRPGONameForGlobalObject(GO, GO.getLinkage(), FileName);
+ }
+
+ // In LTO mode (when InLTO is true), first check if there is a meta data.
+ if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata))
+ return *IRPGOFuncName;
+
+ // If there is no meta data, the function must be a global before the value
+ // profile annotation pass. Its current linkage may be internal if it is
+ // internalized in LTO mode.
+ return getIRPGONameForGlobalObject(GO, GlobalValue::ExternalLinkage, "");
+}
+
+// Returns the IRPGO function name and does special handling when called
+// in LTO optimization. See the comments of `getIRPGOObjectName` for details.
+std::string getIRPGOFuncName(const Function &F, bool InLTO) {
+ return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
+}
+
+// This is similar to `getIRPGOFuncName` except that this function calls
+// 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
+// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
+// comments of `getIRPGONameForGlobalObject`.
std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
if (!InLTO) {
- StringRef FileName(F.getParent()->getSourceFileName());
- uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
- if (StripLevel < StaticFuncStripDirNamePrefix)
- StripLevel = StaticFuncStripDirNamePrefix;
- if (StripLevel)
- FileName = stripDirPrefix(FileName, StripLevel);
+ auto FileName = getStrippedSourceFileName(F);
return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
}
// In LTO mode (when InLTO is true), first check if there is a meta data.
- if (MDNode *MD = getPGOFuncNameMetadata(F)) {
- StringRef S = cast<MDString>(MD->getOperand(0))->getString();
- return S.str();
- }
+ if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F)))
+ return *PGOFuncName;
// If there is no meta data, the function must be a global before the value
// profile annotation pass. Its current linkage may be internal if it is
@@ -300,6 +372,15 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
}
+// See getIRPGOFuncName() for a discription of the format.
+std::pair<StringRef, StringRef>
+getParsedIRPGOFuncName(StringRef IRPGOFuncName) {
+ auto [FileName, FuncName] = IRPGOFuncName.split(';');
+ if (FuncName.empty())
+ return std::make_pair(StringRef(), IRPGOFuncName);
+ return std::make_pair(FileName, FuncName);
+}
+
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
if (FileName.empty())
return PGOFuncName;
@@ -320,7 +401,7 @@ std::string getPGOFuncNameVarName(StringRef FuncName,
return VarName;
// Now fix up illegal chars in local VarName that may upset the assembler.
- const char *InvalidChars = "-:<>/\"'";
+ const char InvalidChars[] = "-:;<>/\"'";
size_t found = VarName.find_first_of(InvalidChars);
while (found != std::string::npos) {
VarName[found] = '_';
@@ -366,41 +447,102 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
// Ignore in this case.
if (!F.hasName())
continue;
- const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
- if (Error E = addFuncName(PGOFuncName))
+ if (Error E = addFuncWithName(F, getIRPGOFuncName(F, InLTO)))
+ return E;
+ // Also use getPGOFuncName() so that we can find records from older profiles
+ if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO)))
return E;
- MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
- // In ThinLTO, local function may have been promoted to global and have
- // suffix ".llvm." added to the function name. We need to add the
- // stripped function name to the symbol table so that we can find a match
- // from profile.
- //
- // We may have other suffixes similar as ".llvm." which are needed to
- // be stripped before the matching, but ".__uniq." suffix which is used
- // to differentiate internal linkage functions in different modules
- // should be kept. Now this is the only suffix with the pattern ".xxx"
- // which is kept before matching.
- const std::string UniqSuffix = ".__uniq.";
- auto pos = PGOFuncName.find(UniqSuffix);
- // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
- // search '.' from the beginning.
- if (pos != std::string::npos)
- pos += UniqSuffix.length();
- else
- pos = 0;
- pos = PGOFuncName.find('.', pos);
- if (pos != std::string::npos && pos != 0) {
- const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
- if (Error E = addFuncName(OtherFuncName))
- return E;
- MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
- }
}
Sorted = false;
finalizeSymtab();
return Error::success();
}
+/// \c NameStrings is a string composed of one of more possibly encoded
+/// sub-strings. The substrings are separated by 0 or more zero bytes. This
+/// method decodes the string and calls `NameCallback` for each substring.
+static Error
+readAndDecodeStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback) {
+ const uint8_t *P = NameStrings.bytes_begin();
+ const uint8_t *EndP = NameStrings.bytes_end();
+ while (P < EndP) {
+ uint32_t N;
+ uint64_t UncompressedSize = decodeULEB128(P, &N);
+ P += N;
+ uint64_t CompressedSize = decodeULEB128(P, &N);
+ P += N;
+ bool isCompressed = (CompressedSize != 0);
+ SmallVector<uint8_t, 128> UncompressedNameStrings;
+ StringRef NameStrings;
+ if (isCompressed) {
+ if (!llvm::compression::zlib::isAvailable())
+ return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
+
+ if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
+ UncompressedNameStrings,
+ UncompressedSize)) {
+ consumeError(std::move(E));
+ return make_error<InstrProfError>(instrprof_error::uncompress_failed);
+ }
+ P += CompressedSize;
+ NameStrings = toStringRef(UncompressedNameStrings);
+ } else {
+ NameStrings =
+ StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
+ P += UncompressedSize;
+ }
+ // Now parse the name strings.
+ SmallVector<StringRef, 0> Names;
+ NameStrings.split(Names, getInstrProfNameSeparator());
+ for (StringRef &Name : Names)
+ if (Error E = NameCallback(Name))
+ return E;
+
+ while (P < EndP && *P == 0)
+ P++;
+ }
+ return Error::success();
+}
+
+Error InstrProfSymtab::create(StringRef NameStrings) {
+ return readAndDecodeStrings(
+ NameStrings,
+ std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
+}
+
+Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
+ if (Error E = addFuncName(PGOFuncName))
+ return E;
+ MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
+ // In ThinLTO, local function may have been promoted to global and have
+ // suffix ".llvm." added to the function name. We need to add the
+ // stripped function name to the symbol table so that we can find a match
+ // from profile.
+ //
+ // We may have other suffixes similar as ".llvm." which are needed to
+ // be stripped before the matching, but ".__uniq." suffix which is used
+ // to differentiate internal linkage functions in different modules
+ // should be kept. Now this is the only suffix with the pattern ".xxx"
+ // which is kept before matching.
+ const std::string UniqSuffix = ".__uniq.";
+ auto pos = PGOFuncName.find(UniqSuffix);
+ // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
+ // search '.' from the beginning.
+ if (pos != std::string::npos)
+ pos += UniqSuffix.length();
+ else
+ pos = 0;
+ pos = PGOFuncName.find('.', pos);
+ if (pos != std::string::npos && pos != 0) {
+ StringRef OtherFuncName = PGOFuncName.substr(0, pos);
+ if (Error E = addFuncName(OtherFuncName))
+ return E;
+ MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
+ }
+ return Error::success();
+}
+
uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
finalizeSymtab();
auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
@@ -422,11 +564,11 @@ void InstrProfSymtab::dumpNames(raw_ostream &OS) const {
OS << S << '\n';
}
-Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
- bool doCompression, std::string &Result) {
+Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
+ bool doCompression, std::string &Result) {
assert(!NameStrs.empty() && "No name data to emit");
- uint8_t Header[16], *P = Header;
+ uint8_t Header[20], *P = Header;
std::string UncompressedNameStrings =
join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
@@ -473,52 +615,10 @@ Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
for (auto *NameVar : NameVars) {
NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
}
- return collectPGOFuncNameStrings(
+ return collectGlobalObjectNameStrings(
NameStrs, compression::zlib::isAvailable() && doCompression, Result);
}
-Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
- const uint8_t *P = NameStrings.bytes_begin();
- const uint8_t *EndP = NameStrings.bytes_end();
- while (P < EndP) {
- uint32_t N;
- uint64_t UncompressedSize = decodeULEB128(P, &N);
- P += N;
- uint64_t CompressedSize = decodeULEB128(P, &N);
- P += N;
- bool isCompressed = (CompressedSize != 0);
- SmallVector<uint8_t, 128> UncompressedNameStrings;
- StringRef NameStrings;
- if (isCompressed) {
- if (!llvm::compression::zlib::isAvailable())
- return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
-
- if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
- UncompressedNameStrings,
- UncompressedSize)) {
- consumeError(std::move(E));
- return make_error<InstrProfError>(instrprof_error::uncompress_failed);
- }
- P += CompressedSize;
- NameStrings = toStringRef(UncompressedNameStrings);
- } else {
- NameStrings =
- StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
- P += UncompressedSize;
- }
- // Now parse the name strings.
- SmallVector<StringRef, 0> Names;
- NameStrings.split(Names, getInstrProfNameSeparator());
- for (StringRef &Name : Names)
- if (Error E = Symtab.addFuncName(Name))
- return E;
-
- while (P < EndP && *P == 0)
- P++;
- }
- return Error::success();
-}
-
void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
uint64_t FuncSum = 0;
Sum.NumEntries += Counts.size();
@@ -732,6 +832,18 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
Warn(instrprof_error::counter_overflow);
}
+ // If the number of bitmap bytes doesn't match we either have bad data
+ // or a hash collision.
+ if (BitmapBytes.size() != Other.BitmapBytes.size()) {
+ Warn(instrprof_error::bitmap_mismatch);
+ return;
+ }
+
+ // Bitmap bytes are merged by simply ORing them together.
+ for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) {
+ BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I];
+ }
+
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
mergeValueProfData(Kind, Other, Weight, Warn);
}
@@ -910,14 +1022,13 @@ void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
// For writing/serializing, Old is the host endianness, and New is
// byte order intended on disk. For Reading/deserialization, Old
// is the on-disk source endianness, and New is the host endianness.
-void ValueProfRecord::swapBytes(support::endianness Old,
- support::endianness New) {
+void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) {
using namespace support;
if (Old == New)
return;
- if (getHostEndianness() != Old) {
+ if (llvm::endianness::native != Old) {
sys::swapByteOrder<uint32_t>(NumValueSites);
sys::swapByteOrder<uint32_t>(Kind);
}
@@ -929,7 +1040,7 @@ void ValueProfRecord::swapBytes(support::endianness Old,
sys::swapByteOrder<uint64_t>(VD[I].Value);
sys::swapByteOrder<uint64_t>(VD[I].Count);
}
- if (getHostEndianness() == Old) {
+ if (llvm::endianness::native == Old) {
sys::swapByteOrder<uint32_t>(NumValueSites);
sys::swapByteOrder<uint32_t>(Kind);
}
@@ -948,13 +1059,13 @@ void ValueProfData::deserializeTo(InstrProfRecord &Record,
}
template <class T>
-static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
+static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig) {
using namespace support;
- if (Orig == little)
- return endian::readNext<T, little, unaligned>(D);
+ if (Orig == llvm::endianness::little)
+ return endian::readNext<T, llvm::endianness::little, unaligned>(D);
else
- return endian::readNext<T, big, unaligned>(D);
+ return endian::readNext<T, llvm::endianness::big, unaligned>(D);
}
static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
@@ -988,7 +1099,7 @@ Error ValueProfData::checkIntegrity() {
Expected<std::unique_ptr<ValueProfData>>
ValueProfData::getValueProfData(const unsigned char *D,
const unsigned char *const BufferEnd,
- support::endianness Endianness) {
+ llvm::endianness Endianness) {
using namespace support;
if (D + sizeof(ValueProfData) > BufferEnd)
@@ -1011,10 +1122,10 @@ ValueProfData::getValueProfData(const unsigned char *D,
return std::move(VPD);
}
-void ValueProfData::swapBytesToHost(support::endianness Endianness) {
+void ValueProfData::swapBytesToHost(llvm::endianness Endianness) {
using namespace support;
- if (Endianness == getHostEndianness())
+ if (Endianness == llvm::endianness::native)
return;
sys::swapByteOrder<uint32_t>(TotalSize);
@@ -1022,21 +1133,21 @@ void ValueProfData::swapBytesToHost(support::endianness Endianness) {
ValueProfRecord *VR = getFirstValueProfRecord(this);
for (uint32_t K = 0; K < NumValueKinds; K++) {
- VR->swapBytes(Endianness, getHostEndianness());
+ VR->swapBytes(Endianness, llvm::endianness::native);
VR = getValueProfRecordNext(VR);
}
}
-void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
+void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) {
using namespace support;
- if (Endianness == getHostEndianness())
+ if (Endianness == llvm::endianness::native)
return;
ValueProfRecord *VR = getFirstValueProfRecord(this);
for (uint32_t K = 0; K < NumValueKinds; K++) {
ValueProfRecord *NVR = getValueProfRecordNext(VR);
- VR->swapBytes(getHostEndianness(), Endianness);
+ VR->swapBytes(llvm::endianness::native, Endianness);
VR = NVR;
}
sys::swapByteOrder<uint32_t>(TotalSize);
@@ -1378,7 +1489,7 @@ static inline uint64_t read(const unsigned char *Buffer, size_t Offset) {
uint64_t Header::formatVersion() const {
using namespace support;
- return endian::byte_swap<uint64_t, little>(Version);
+ return endian::byte_swap<uint64_t, llvm::endianness::little>(Version);
}
Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
@@ -1390,7 +1501,8 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
H.Magic = read(Buffer, offsetOf(&Header::Magic));
// Check the magic number.
- uint64_t Magic = endian::byte_swap<uint64_t, little>(H.Magic);
+ uint64_t Magic =
+ endian::byte_swap<uint64_t, llvm::endianness::little>(H.Magic);
if (Magic != IndexedInstrProf::Magic)
return make_error<InstrProfError>(instrprof_error::bad_magic);
@@ -1404,9 +1516,11 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
// When a new field is added in the header add a case statement here to
// populate it.
static_assert(
- IndexedInstrProf::ProfVersion::CurrentVersion == Version10,
+ IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
"Please update the reading code below if a new field has been added, "
"if not add a case statement to fall through to the latest version.");
+ case 11ull:
+ [[fallthrough]];
case 10ull:
H.TemporalProfTracesOffset =
read(Buffer, offsetOf(&Header::TemporalProfTracesOffset));
@@ -1430,10 +1544,12 @@ size_t Header::size() const {
// When a new field is added to the header add a case statement here to
// compute the size as offset of the new field + size of the new field. This
// relies on the field being added to the end of the list.
- static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10,
+ static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11,
"Please update the size computation below if a new field has "
"been added to the header, if not add a case statement to "
"fall through to the latest version.");
+ case 11ull:
+ [[fallthrough]];
case 10ull:
return offsetOf(&Header::TemporalProfTracesOffset) +
sizeof(Header::TemporalProfTracesOffset);
generated by cgit v1.3 (git 2.53.0) at 2026-03-10 14:32:54 +0000