summaryrefslogtreecommitdiff
path: root/lib/Bitcode
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Bitcode')
-rw-r--r--lib/Bitcode/Reader/BitcodeReader.cpp193
-rw-r--r--lib/Bitcode/Reader/MetadataLoader.cpp74
-rw-r--r--lib/Bitcode/Reader/ValueList.cpp6
-rw-r--r--lib/Bitcode/Writer/BitWriter.cpp6
-rw-r--r--lib/Bitcode/Writer/BitcodeWriter.cpp228
-rw-r--r--lib/Bitcode/Writer/BitcodeWriterPass.cpp8
-rw-r--r--lib/Bitcode/Writer/ValueEnumerator.cpp7
7 files changed, 407 insertions, 115 deletions
diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp
index 95291a1caf9a..c45b441238bc 100644
--- a/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -23,6 +23,7 @@
#include "llvm/ADT/Twine.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Bitcode/LLVMBitCodes.h"
+#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/AutoUpgrade.h"
@@ -532,7 +533,7 @@ public:
Error materializeModule() override;
std::vector<StructType *> getIdentifiedStructTypes() const override;
- /// \brief Main interface to parsing a bitcode buffer.
+ /// Main interface to parsing a bitcode buffer.
/// \returns true if an error occurred.
Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata = false,
bool IsImporting = false);
@@ -743,14 +744,16 @@ private:
std::vector<ValueInfo> makeRefList(ArrayRef<uint64_t> Record);
std::vector<FunctionSummary::EdgeTy> makeCallList(ArrayRef<uint64_t> Record,
bool IsOldProfileFormat,
- bool HasProfile);
+ bool HasProfile,
+ bool HasRelBF);
Error parseEntireSummary(unsigned ID);
Error parseModuleStringTable();
std::pair<ValueInfo, GlobalValue::GUID>
getValueInfoFromValueId(unsigned ValueId);
- ModuleSummaryIndex::ModuleInfo *addThisModule();
+ void addThisModule();
+ ModuleSummaryIndex::ModuleInfo *getThisModule();
};
} // end anonymous namespace
@@ -1046,19 +1049,21 @@ static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) {
static FastMathFlags getDecodedFastMathFlags(unsigned Val) {
FastMathFlags FMF;
- if (0 != (Val & FastMathFlags::AllowReassoc))
+ if (0 != (Val & bitc::UnsafeAlgebra))
+ FMF.setFast();
+ if (0 != (Val & bitc::AllowReassoc))
FMF.setAllowReassoc();
- if (0 != (Val & FastMathFlags::NoNaNs))
+ if (0 != (Val & bitc::NoNaNs))
FMF.setNoNaNs();
- if (0 != (Val & FastMathFlags::NoInfs))
+ if (0 != (Val & bitc::NoInfs))
FMF.setNoInfs();
- if (0 != (Val & FastMathFlags::NoSignedZeros))
+ if (0 != (Val & bitc::NoSignedZeros))
FMF.setNoSignedZeros();
- if (0 != (Val & FastMathFlags::AllowReciprocal))
+ if (0 != (Val & bitc::AllowReciprocal))
FMF.setAllowReciprocal();
- if (0 != (Val & FastMathFlags::AllowContract))
+ if (0 != (Val & bitc::AllowContract))
FMF.setAllowContract(true);
- if (0 != (Val & FastMathFlags::ApproxFunc))
+ if (0 != (Val & bitc::ApproxFunc))
FMF.setApproxFunc();
return FMF;
}
@@ -1157,6 +1162,9 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) {
case Attribute::Speculatable: return 1ULL << 54;
case Attribute::StrictFP: return 1ULL << 55;
case Attribute::SanitizeHWAddress: return 1ULL << 56;
+ case Attribute::NoCfCheck: return 1ULL << 57;
+ case Attribute::OptForFuzzing: return 1ULL << 58;
+ case Attribute::ShadowCallStack: return 1ULL << 59;
case Attribute::Dereferenceable:
llvm_unreachable("dereferenceable attribute not supported in raw format");
break;
@@ -1195,7 +1203,7 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) {
}
}
-/// \brief This fills an AttrBuilder object with the LLVM attributes that have
+/// This fills an AttrBuilder object with the LLVM attributes that have
/// been decoded from the given integer. This function must stay in sync with
/// 'encodeLLVMAttributesForBitcode'.
static void decodeLLVMAttributesForBitcode(AttrBuilder &B,
@@ -1335,8 +1343,12 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::NoRedZone;
case bitc::ATTR_KIND_NO_RETURN:
return Attribute::NoReturn;
+ case bitc::ATTR_KIND_NOCF_CHECK:
+ return Attribute::NoCfCheck;
case bitc::ATTR_KIND_NO_UNWIND:
return Attribute::NoUnwind;
+ case bitc::ATTR_KIND_OPT_FOR_FUZZING:
+ return Attribute::OptForFuzzing;
case bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE:
return Attribute::OptimizeForSize;
case bitc::ATTR_KIND_OPTIMIZE_NONE:
@@ -1363,6 +1375,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::StackProtectStrong;
case bitc::ATTR_KIND_SAFESTACK:
return Attribute::SafeStack;
+ case bitc::ATTR_KIND_SHADOWCALLSTACK:
+ return Attribute::ShadowCallStack;
case bitc::ATTR_KIND_STRICT_FP:
return Attribute::StrictFP;
case bitc::ATTR_KIND_STRUCT_RET:
@@ -2510,6 +2524,7 @@ Error BitcodeReader::parseConstants() {
for (unsigned i = 0; i != ConstStrSize; ++i)
ConstrStr += (char)Record[3+AsmStrSize+i];
PointerType *PTy = cast<PointerType>(CurTy);
+ UpgradeInlineAsmString(&AsmStr);
V = InlineAsm::get(cast<FunctionType>(PTy->getElementType()),
AsmStr, ConstrStr, HasSideEffects, IsAlignStack);
break;
@@ -2535,6 +2550,7 @@ Error BitcodeReader::parseConstants() {
for (unsigned i = 0; i != ConstStrSize; ++i)
ConstrStr += (char)Record[3+AsmStrSize+i];
PointerType *PTy = cast<PointerType>(CurTy);
+ UpgradeInlineAsmString(&AsmStr);
V = InlineAsm::get(cast<FunctionType>(PTy->getElementType()),
AsmStr, ConstrStr, HasSideEffects, IsAlignStack,
InlineAsm::AsmDialect(AsmDialect));
@@ -2815,6 +2831,13 @@ Error BitcodeReader::parseComdatRecord(ArrayRef<uint64_t> Record) {
return Error::success();
}
+static void inferDSOLocal(GlobalValue *GV) {
+ // infer dso_local from linkage and visibility if it is not encoded.
+ if (GV->hasLocalLinkage() ||
+ (!GV->hasDefaultVisibility() && !GV->hasExternalWeakLinkage()))
+ GV->setDSOLocal(true);
+}
+
Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) {
// v1: [pointer type, isconst, initid, linkage, alignment, section,
// visibility, threadlocal, unnamed_addr, externally_initialized,
@@ -2907,6 +2930,7 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) {
if (Record.size() > 13) {
NewGV->setDSOLocal(getDecodedDSOLocal(Record[13]));
}
+ inferDSOLocal(NewGV);
return Error::success();
}
@@ -2991,6 +3015,7 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
if (Record.size() > 15) {
Func->setDSOLocal(getDecodedDSOLocal(Record[15]));
}
+ inferDSOLocal(Func);
ValueList.push_back(Func);
@@ -3054,16 +3079,21 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord(
// FIXME: Change to an error if non-default in 4.0.
NewGA->setVisibility(getDecodedVisibility(Record[VisInd]));
}
- if (OpNum != Record.size())
- NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++]));
- else
- upgradeDLLImportExportLinkage(NewGA, Linkage);
- if (OpNum != Record.size())
- NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++]));
- if (OpNum != Record.size())
- NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++]));
+ if (BitCode == bitc::MODULE_CODE_ALIAS ||
+ BitCode == bitc::MODULE_CODE_ALIAS_OLD) {
+ if (OpNum != Record.size())
+ NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++]));
+ else
+ upgradeDLLImportExportLinkage(NewGA, Linkage);
+ if (OpNum != Record.size())
+ NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++]));
+ if (OpNum != Record.size())
+ NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++]));
+ }
if (OpNum != Record.size())
NewGA->setDSOLocal(getDecodedDSOLocal(Record[OpNum++]));
+ inferDSOLocal(NewGA);
+
ValueList.push_back(NewGA);
IndirectSymbolInits.push_back(std::make_pair(NewGA, Val));
return Error::success();
@@ -4773,6 +4803,9 @@ Error BitcodeReader::materializeModule() {
UpgradeDebugInfo(*TheModule);
UpgradeModuleFlags(*TheModule);
+
+ UpgradeRetainReleaseMarker(*TheModule);
+
return Error::success();
}
@@ -4786,9 +4819,13 @@ ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader(
: BitcodeReaderBase(std::move(Cursor), Strtab), TheIndex(TheIndex),
ModulePath(ModulePath), ModuleId(ModuleId) {}
+void ModuleSummaryIndexBitcodeReader::addThisModule() {
+ TheIndex.addModule(ModulePath, ModuleId);
+}
+
ModuleSummaryIndex::ModuleInfo *
-ModuleSummaryIndexBitcodeReader::addThisModule() {
- return TheIndex.addModule(ModulePath, ModuleId);
+ModuleSummaryIndexBitcodeReader::getThisModule() {
+ return TheIndex.getModule(ModulePath);
}
std::pair<ValueInfo, GlobalValue::GUID>
@@ -4810,8 +4847,15 @@ void ModuleSummaryIndexBitcodeReader::setValueGUID(
if (PrintSummaryGUIDs)
dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is "
<< ValueName << "\n";
- ValueIdToValueInfoMap[ValueID] =
- std::make_pair(TheIndex.getOrInsertValueInfo(ValueGUID), OriginalNameID);
+
+ // UseStrtab is false for legacy summary formats and value names are
+ // created on stack. In that case we save the name in a string saver in
+ // the index so that the value name can be recorded.
+ ValueIdToValueInfoMap[ValueID] = std::make_pair(
+ TheIndex.getOrInsertValueInfo(
+ ValueGUID,
+ UseStrtab ? ValueName : TheIndex.saveString(ValueName.str())),
+ OriginalNameID);
}
// Specialized value symbol table parser used when reading module index
@@ -4940,6 +4984,9 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() {
break;
case bitc::GLOBALVAL_SUMMARY_BLOCK_ID:
case bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID:
+ // Add the module if it is a per-module index (has a source file name).
+ if (!SourceFileName.empty())
+ addThisModule();
assert(!SeenValueSymbolTable &&
"Already read VST when parsing summary block?");
// We might not have a VST if there were no values in the
@@ -4985,7 +5032,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() {
case bitc::MODULE_CODE_HASH: {
if (Record.size() != 5)
return error("Invalid hash length " + Twine(Record.size()).str());
- auto &Hash = addThisModule()->second.second;
+ auto &Hash = getThisModule()->second.second;
int Pos = 0;
for (auto &Val : Record) {
assert(!(Val >> 32) && "Unexpected high bits set");
@@ -5040,12 +5087,15 @@ ModuleSummaryIndexBitcodeReader::makeRefList(ArrayRef<uint64_t> Record) {
return Ret;
}
-std::vector<FunctionSummary::EdgeTy> ModuleSummaryIndexBitcodeReader::makeCallList(
- ArrayRef<uint64_t> Record, bool IsOldProfileFormat, bool HasProfile) {
+std::vector<FunctionSummary::EdgeTy>
+ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef<uint64_t> Record,
+ bool IsOldProfileFormat,
+ bool HasProfile, bool HasRelBF) {
std::vector<FunctionSummary::EdgeTy> Ret;
Ret.reserve(Record.size());
for (unsigned I = 0, E = Record.size(); I != E; ++I) {
CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown;
+ uint64_t RelBF = 0;
ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
if (IsOldProfileFormat) {
I += 1; // Skip old callsitecount field
@@ -5053,11 +5103,63 @@ std::vector<FunctionSummary::EdgeTy> ModuleSummaryIndexBitcodeReader::makeCallLi
I += 1; // Skip old profilecount field
} else if (HasProfile)
Hotness = static_cast<CalleeInfo::HotnessType>(Record[++I]);
- Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}});
+ else if (HasRelBF)
+ RelBF = Record[++I];
+ Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo(Hotness, RelBF)});
}
return Ret;
}
+static void
+parseWholeProgramDevirtResolutionByArg(ArrayRef<uint64_t> Record, size_t &Slot,
+ WholeProgramDevirtResolution &Wpd) {
+ uint64_t ArgNum = Record[Slot++];
+ WholeProgramDevirtResolution::ByArg &B =
+ Wpd.ResByArg[{Record.begin() + Slot, Record.begin() + Slot + ArgNum}];
+ Slot += ArgNum;
+
+ B.TheKind =
+ static_cast<WholeProgramDevirtResolution::ByArg::Kind>(Record[Slot++]);
+ B.Info = Record[Slot++];
+ B.Byte = Record[Slot++];
+ B.Bit = Record[Slot++];
+}
+
+static void parseWholeProgramDevirtResolution(ArrayRef<uint64_t> Record,
+ StringRef Strtab, size_t &Slot,
+ TypeIdSummary &TypeId) {
+ uint64_t Id = Record[Slot++];
+ WholeProgramDevirtResolution &Wpd = TypeId.WPDRes[Id];
+
+ Wpd.TheKind = static_cast<WholeProgramDevirtResolution::Kind>(Record[Slot++]);
+ Wpd.SingleImplName = {Strtab.data() + Record[Slot],
+ static_cast<size_t>(Record[Slot + 1])};
+ Slot += 2;
+
+ uint64_t ResByArgNum = Record[Slot++];
+ for (uint64_t I = 0; I != ResByArgNum; ++I)
+ parseWholeProgramDevirtResolutionByArg(Record, Slot, Wpd);
+}
+
+static void parseTypeIdSummaryRecord(ArrayRef<uint64_t> Record,
+ StringRef Strtab,
+ ModuleSummaryIndex &TheIndex) {
+ size_t Slot = 0;
+ TypeIdSummary &TypeId = TheIndex.getOrInsertTypeIdSummary(
+ {Strtab.data() + Record[Slot], static_cast<size_t>(Record[Slot + 1])});
+ Slot += 2;
+
+ TypeId.TTRes.TheKind = static_cast<TypeTestResolution::Kind>(Record[Slot++]);
+ TypeId.TTRes.SizeM1BitWidth = Record[Slot++];
+ TypeId.TTRes.AlignLog2 = Record[Slot++];
+ TypeId.TTRes.SizeM1 = Record[Slot++];
+ TypeId.TTRes.BitMask = Record[Slot++];
+ TypeId.TTRes.InlineBits = Record[Slot++];
+
+ while (Slot < Record.size())
+ parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId);
+}
+
// Eagerly parse the entire summary block. This populates the GlobalValueSummary
// objects in the index.
Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
@@ -5120,6 +5222,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
switch (BitCode) {
default: // Default behavior: ignore.
break;
+ case bitc::FS_FLAGS: { // [flags]
+ uint64_t Flags = Record[0];
+ // Scan flags (set only on the combined index).
+ assert(Flags <= 0x3 && "Unexpected bits in flag");
+
+ // 1 bit: WithGlobalValueDeadStripping flag.
+ if (Flags & 0x1)
+ TheIndex.setWithGlobalValueDeadStripping();
+ // 1 bit: SkipModuleByDistributedBackend flag.
+ if (Flags & 0x2)
+ TheIndex.setSkipModuleByDistributedBackend();
+ break;
+ }
case bitc::FS_VALUE_GUID: { // [valueid, refguid]
uint64_t ValueID = Record[0];
GlobalValue::GUID RefGUID = Record[1];
@@ -5132,7 +5247,11 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
// FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs,
// numrefs x valueid,
// n x (valueid, hotness)]
+ // FS_PERMODULE_RELBF: [valueid, flags, instcount, fflags, numrefs,
+ // numrefs x valueid,
+ // n x (valueid, relblockfreq)]
case bitc::FS_PERMODULE:
+ case bitc::FS_PERMODULE_RELBF:
case bitc::FS_PERMODULE_PROFILE: {
unsigned ValueID = Record[0];
uint64_t RawFlags = Record[1];
@@ -5158,9 +5277,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::vector<ValueInfo> Refs = makeRefList(
ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs));
bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE);
+ bool HasRelBF = (BitCode == bitc::FS_PERMODULE_RELBF);
std::vector<FunctionSummary::EdgeTy> Calls = makeCallList(
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
- IsOldProfileFormat, HasProfile);
+ IsOldProfileFormat, HasProfile, HasRelBF);
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs),
std::move(Calls), std::move(PendingTypeTests),
@@ -5174,7 +5294,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
PendingTypeTestAssumeConstVCalls.clear();
PendingTypeCheckedLoadConstVCalls.clear();
auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID);
- FS->setModulePath(addThisModule()->first());
+ FS->setModulePath(getThisModule()->first());
FS->setOriginalName(VIAndOriginalGUID.second);
TheIndex.addGlobalValueSummary(VIAndOriginalGUID.first, std::move(FS));
break;
@@ -5193,7 +5313,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
// string table section in the per-module index, we create a single
// module path string table entry with an empty (0) ID to take
// ownership.
- AS->setModulePath(addThisModule()->first());
+ AS->setModulePath(getThisModule()->first());
GlobalValue::GUID AliaseeGUID =
getValueInfoFromValueId(AliaseeID).first.getGUID();
@@ -5217,7 +5337,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::vector<ValueInfo> Refs =
makeRefList(ArrayRef<uint64_t>(Record).slice(2));
auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs));
- FS->setModulePath(addThisModule()->first());
+ FS->setModulePath(getThisModule()->first());
auto GUID = getValueInfoFromValueId(ValueID);
FS->setOriginalName(GUID.second);
TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
@@ -5252,7 +5372,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE);
std::vector<FunctionSummary::EdgeTy> Edges = makeCallList(
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
- IsOldProfileFormat, HasProfile);
+ IsOldProfileFormat, HasProfile, false);
ValueInfo VI = getValueInfoFromValueId(ValueID).first;
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs),
@@ -5360,6 +5480,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
{Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])});
break;
}
+
case bitc::FS_CFI_FUNCTION_DECLS: {
std::set<std::string> &CfiFunctionDecls = TheIndex.cfiFunctionDecls();
for (unsigned I = 0; I != Record.size(); I += 2)
@@ -5367,6 +5488,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
{Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])});
break;
}
+
+ case bitc::FS_TYPE_ID:
+ parseTypeIdSummaryRecord(Record, Strtab, TheIndex);
+ break;
}
}
llvm_unreachable("Exit infinite loop");
@@ -5602,7 +5727,7 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) {
}
}
-/// \brief Get a lazy one-at-time loading module from bitcode.
+/// Get a lazy one-at-time loading module from bitcode.
///
/// This isn't always used in a lazy context. In particular, it's also used by
/// \a parseModule(). If this is truly lazy, then we need to eagerly pull
@@ -5676,7 +5801,7 @@ Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() {
BitstreamCursor Stream(Buffer);
Stream.JumpToBit(ModuleBit);
- auto Index = llvm::make_unique<ModuleSummaryIndex>();
+ auto Index = llvm::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, *Index,
ModuleIdentifier, 0);
diff --git a/lib/Bitcode/Reader/MetadataLoader.cpp b/lib/Bitcode/Reader/MetadataLoader.cpp
index 1b0d80d26cf5..011c41e2cecd 100644
--- a/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -165,7 +165,7 @@ public:
/// necessary.
Metadata *getMetadataFwdRef(unsigned Idx);
- /// Return the the given metadata only if it is fully resolved.
+ /// Return the given metadata only if it is fully resolved.
///
/// Gives the same result as \a lookup(), unless \a MDNode::isResolved()
/// would give \c false.
@@ -822,6 +822,7 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() {
case bitc::METADATA_TEMPLATE_VALUE:
case bitc::METADATA_GLOBAL_VAR:
case bitc::METADATA_LOCAL_VAR:
+ case bitc::METADATA_LABEL:
case bitc::METADATA_EXPRESSION:
case bitc::METADATA_OBJC_PROPERTY:
case bitc::METADATA_IMPORTED_ENTITY:
@@ -1174,14 +1175,25 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
break;
}
case bitc::METADATA_SUBRANGE: {
- if (Record.size() != 3)
- return error("Invalid record");
+ Metadata *Val = nullptr;
+ // Operand 'count' is interpreted as:
+ // - Signed integer (version 0)
+ // - Metadata node (version 1)
+ switch (Record[0] >> 1) {
+ case 0:
+ Val = GET_OR_DISTINCT(DISubrange,
+ (Context, Record[1], unrotateSign(Record.back())));
+ break;
+ case 1:
+ Val = GET_OR_DISTINCT(DISubrange, (Context, getMDOrNull(Record[1]),
+ unrotateSign(Record.back())));
+ break;
+ default:
+ return error("Invalid record: Unsupported version of DISubrange");
+ }
- IsDistinct = Record[0];
- MetadataList.assignValue(
- GET_OR_DISTINCT(DISubrange,
- (Context, Record[1], unrotateSign(Record[2]))),
- NextMetadataNo);
+ MetadataList.assignValue(Val, NextMetadataNo);
+ IsDistinct = Record[0] & 1;
NextMetadataNo++;
break;
}
@@ -1189,10 +1201,11 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
if (Record.size() != 3)
return error("Invalid record");
- IsDistinct = Record[0];
+ IsDistinct = Record[0] & 1;
+ bool IsUnsigned = Record[0] & 2;
MetadataList.assignValue(
GET_OR_DISTINCT(DIEnumerator, (Context, unrotateSign(Record[1]),
- getMDString(Record[2]))),
+ IsUnsigned, getMDString(Record[2]))),
NextMetadataNo);
NextMetadataNo++;
break;
@@ -1235,7 +1248,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
break;
}
case bitc::METADATA_COMPOSITE_TYPE: {
- if (Record.size() != 16)
+ if (Record.size() < 16 || Record.size() > 17)
return error("Invalid record");
// If we have a UUID and this is not a forward declaration, lookup the
@@ -1258,6 +1271,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
unsigned RuntimeLang = Record[12];
Metadata *VTableHolder = nullptr;
Metadata *TemplateParams = nullptr;
+ Metadata *Discriminator = nullptr;
auto *Identifier = getMDString(Record[15]);
// If this module is being parsed so that it can be ThinLTO imported
// into another module, composite types only need to be imported
@@ -1278,13 +1292,15 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
Elements = getMDOrNull(Record[11]);
VTableHolder = getDITypeRefOrNull(Record[13]);
TemplateParams = getMDOrNull(Record[14]);
+ if (Record.size() > 16)
+ Discriminator = getMDOrNull(Record[16]);
}
DICompositeType *CT = nullptr;
if (Identifier)
CT = DICompositeType::buildODRType(
Context, *Identifier, Tag, Name, File, Line, Scope, BaseType,
SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
- VTableHolder, TemplateParams);
+ VTableHolder, TemplateParams, Discriminator);
// Create a node if we didn't get a lazy ODR type.
if (!CT)
@@ -1335,17 +1351,25 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
}
case bitc::METADATA_FILE: {
- if (Record.size() != 3 && Record.size() != 5)
+ if (Record.size() != 3 && Record.size() != 5 && Record.size() != 6)
return error("Invalid record");
IsDistinct = Record[0];
+ Optional<DIFile::ChecksumInfo<MDString *>> Checksum;
+ // The BitcodeWriter writes null bytes into Record[3:4] when the Checksum
+ // is not present. This matches up with the old internal representation,
+ // and the old encoding for CSK_None in the ChecksumKind. The new
+ // representation reserves the value 0 in the ChecksumKind to continue to
+ // encode None in a backwards-compatible way.
+ if (Record.size() > 4 && Record[3] && Record[4])
+ Checksum.emplace(static_cast<DIFile::ChecksumKind>(Record[3]),
+ getMDString(Record[4]));
MetadataList.assignValue(
GET_OR_DISTINCT(
DIFile,
- (Context, getMDString(Record[1]), getMDString(Record[2]),
- Record.size() == 3 ? DIFile::CSK_None
- : static_cast<DIFile::ChecksumKind>(Record[3]),
- Record.size() == 3 ? nullptr : getMDString(Record[4]))),
+ (Context, getMDString(Record[1]), getMDString(Record[2]), Checksum,
+ Record.size() > 5 ? Optional<MDString *>(getMDString(Record[5]))
+ : None)),
NextMetadataNo);
NextMetadataNo++;
break;
@@ -1415,7 +1439,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
HasUnit ? CUorFn : nullptr, // unit
getMDOrNull(Record[15 + Offset]), // templateParams
getMDOrNull(Record[16 + Offset]), // declaration
- getMDOrNull(Record[17 + Offset]), // variables
+ getMDOrNull(Record[17 + Offset]), // retainedNodes
HasThrownTypes ? getMDOrNull(Record[20]) : nullptr // thrownTypes
));
MetadataList.assignValue(SP, NextMetadataNo);
@@ -1624,6 +1648,20 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
NextMetadataNo++;
break;
}
+ case bitc::METADATA_LABEL: {
+ if (Record.size() != 5)
+ return error("Invalid record");
+
+ IsDistinct = Record[0] & 1;
+ MetadataList.assignValue(
+ GET_OR_DISTINCT(DILabel,
+ (Context, getMDOrNull(Record[1]),
+ getMDString(Record[2]),
+ getMDOrNull(Record[3]), Record[4])),
+ NextMetadataNo);
+ NextMetadataNo++;
+ break;
+ }
case bitc::METADATA_EXPRESSION: {
if (Record.size() < 1)
return error("Invalid record");
diff --git a/lib/Bitcode/Reader/ValueList.cpp b/lib/Bitcode/Reader/ValueList.cpp
index 08bfa291098c..1ab22b5cc3d1 100644
--- a/lib/Bitcode/Reader/ValueList.cpp
+++ b/lib/Bitcode/Reader/ValueList.cpp
@@ -32,7 +32,7 @@ namespace llvm {
namespace {
-/// \brief A class for maintaining the slot number definition
+/// A class for maintaining the slot number definition
/// as a placeholder for the actual definition for forward constants defs.
class ConstantPlaceHolder : public ConstantExpr {
public:
@@ -46,7 +46,7 @@ public:
// allocate space for exactly one operand
void *operator new(size_t s) { return User::operator new(s, 1); }
- /// \brief Methods to support type inquiry through isa, cast, and dyn_cast.
+ /// Methods to support type inquiry through isa, cast, and dyn_cast.
static bool classof(const Value *V) {
return isa<ConstantExpr>(V) &&
cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1;
@@ -144,7 +144,7 @@ Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) {
void BitcodeReaderValueList::resolveConstantForwardRefs() {
// Sort the values by-pointer so that they are efficient to look up with a
// binary search.
- std::sort(ResolveConstants.begin(), ResolveConstants.end());
+ llvm::sort(ResolveConstants.begin(), ResolveConstants.end());
SmallVector<Constant *, 64> NewOps;
diff --git a/lib/Bitcode/Writer/BitWriter.cpp b/lib/Bitcode/Writer/BitWriter.cpp
index e0388418a3d9..763cd12aa2d7 100644
--- a/lib/Bitcode/Writer/BitWriter.cpp
+++ b/lib/Bitcode/Writer/BitWriter.cpp
@@ -25,7 +25,7 @@ int LLVMWriteBitcodeToFile(LLVMModuleRef M, const char *Path) {
if (EC)
return -1;
- WriteBitcodeToFile(unwrap(M), OS);
+ WriteBitcodeToFile(*unwrap(M), OS);
return 0;
}
@@ -33,7 +33,7 @@ int LLVMWriteBitcodeToFD(LLVMModuleRef M, int FD, int ShouldClose,
int Unbuffered) {
raw_fd_ostream OS(FD, ShouldClose, Unbuffered);
- WriteBitcodeToFile(unwrap(M), OS);
+ WriteBitcodeToFile(*unwrap(M), OS);
return 0;
}
@@ -45,6 +45,6 @@ LLVMMemoryBufferRef LLVMWriteBitcodeToMemoryBuffer(LLVMModuleRef M) {
std::string Data;
raw_string_ostream OS(Data);
- WriteBitcodeToFile(unwrap(M), OS);
+ WriteBitcodeToFile(*unwrap(M), OS);
return wrap(MemoryBuffer::getMemBufferCopy(OS.str()).release());
}
diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp
index a7201ed97350..be75df0820d9 100644
--- a/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -28,6 +28,7 @@
#include "llvm/Bitcode/BitCodes.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Bitcode/LLVMBitCodes.h"
+#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallSite.h"
@@ -86,6 +87,12 @@ static cl::opt<unsigned>
cl::desc("Number of metadatas above which we emit an index "
"to enable lazy-loading"));
+cl::opt<bool> WriteRelBFToSummary(
+ "write-relbf-to-summary", cl::Hidden, cl::init(false),
+ cl::desc("Write relative block frequency to function summary "));
+
+extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
+
namespace {
/// These are manifest constants used by the bitcode writer. They do not need to
@@ -167,12 +174,12 @@ protected:
public:
/// Constructs a ModuleBitcodeWriterBase object for the given Module,
/// writing to the provided \p Buffer.
- ModuleBitcodeWriterBase(const Module *M, StringTableBuilder &StrtabBuilder,
+ ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
BitstreamWriter &Stream,
bool ShouldPreserveUseListOrder,
const ModuleSummaryIndex *Index)
- : BitcodeWriterBase(Stream, StrtabBuilder), M(*M),
- VE(*M, ShouldPreserveUseListOrder), Index(Index) {
+ : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
+ VE(M, ShouldPreserveUseListOrder), Index(Index) {
// Assign ValueIds to any callee values in the index that came from
// indirect call profiles and were recorded as a GUID not a Value*
// (which would have been assigned an ID by the ValueEnumerator).
@@ -190,7 +197,7 @@ public:
// otherwise we would have a Value for it). If so, synthesize
// a value id.
for (auto &CallEdge : FS->calls())
- if (!CallEdge.first.getValue())
+ if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
assignValueId(CallEdge.first.getGUID());
}
@@ -223,7 +230,7 @@ private:
// Helper to get the valueId for the type of value recorded in VI.
unsigned getValueId(ValueInfo VI) {
- if (!VI.getValue())
+ if (!VI.haveGVs() || !VI.getValue())
return getValueId(VI.getGUID());
return VE.getValueID(VI.getValue());
}
@@ -251,7 +258,7 @@ class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
public:
/// Constructs a ModuleBitcodeWriter object for the given Module,
/// writing to the provided \p Buffer.
- ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer,
+ ModuleBitcodeWriter(const Module &M, SmallVectorImpl<char> &Buffer,
StringTableBuilder &StrtabBuilder,
BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
const ModuleSummaryIndex *Index, bool GenerateHash,
@@ -328,6 +335,8 @@ private:
unsigned Abbrev);
void writeDILocalVariable(const DILocalVariable *N,
SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
+ void writeDILabel(const DILabel *N,
+ SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
void writeDIExpression(const DIExpression *N,
SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
@@ -635,8 +644,12 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
return bitc::ATTR_KIND_NO_RED_ZONE;
case Attribute::NoReturn:
return bitc::ATTR_KIND_NO_RETURN;
+ case Attribute::NoCfCheck:
+ return bitc::ATTR_KIND_NOCF_CHECK;
case Attribute::NoUnwind:
return bitc::ATTR_KIND_NO_UNWIND;
+ case Attribute::OptForFuzzing:
+ return bitc::ATTR_KIND_OPT_FOR_FUZZING;
case Attribute::OptimizeForSize:
return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
case Attribute::OptimizeNone:
@@ -663,6 +676,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
case Attribute::SafeStack:
return bitc::ATTR_KIND_SAFESTACK;
+ case Attribute::ShadowCallStack:
+ return bitc::ATTR_KIND_SHADOWCALLSTACK;
case Attribute::StrictFP:
return bitc::ATTR_KIND_STRICT_FP;
case Attribute::StructRet:
@@ -1302,7 +1317,7 @@ void ModuleBitcodeWriter::writeModuleInfo() {
// Emit the ifunc information.
for (const GlobalIFunc &I : M.ifuncs()) {
// IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
- // val#, linkage, visibility]
+ // val#, linkage, visibility, DSO_Local]
Vals.push_back(addToStrtab(I.getName()));
Vals.push_back(I.getName().size());
Vals.push_back(VE.getTypeID(I.getValueType()));
@@ -1310,6 +1325,7 @@ void ModuleBitcodeWriter::writeModuleInfo() {
Vals.push_back(VE.getValueID(I.getResolver()));
Vals.push_back(getEncodedLinkage(I));
Vals.push_back(getEncodedVisibility(I));
+ Vals.push_back(I.isDSOLocal());
Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals);
Vals.clear();
}
@@ -1330,19 +1346,19 @@ static uint64_t getOptimizationFlags(const Value *V) {
Flags |= 1 << bitc::PEO_EXACT;
} else if (const auto *FPMO = dyn_cast<FPMathOperator>(V)) {
if (FPMO->hasAllowReassoc())
- Flags |= FastMathFlags::AllowReassoc;
+ Flags |= bitc::AllowReassoc;
if (FPMO->hasNoNaNs())
- Flags |= FastMathFlags::NoNaNs;
+ Flags |= bitc::NoNaNs;
if (FPMO->hasNoInfs())
- Flags |= FastMathFlags::NoInfs;
+ Flags |= bitc::NoInfs;
if (FPMO->hasNoSignedZeros())
- Flags |= FastMathFlags::NoSignedZeros;
+ Flags |= bitc::NoSignedZeros;
if (FPMO->hasAllowReciprocal())
- Flags |= FastMathFlags::AllowReciprocal;
+ Flags |= bitc::AllowReciprocal;
if (FPMO->hasAllowContract())
- Flags |= FastMathFlags::AllowContract;
+ Flags |= bitc::AllowContract;
if (FPMO->hasApproxFunc())
- Flags |= FastMathFlags::ApproxFunc;
+ Flags |= bitc::ApproxFunc;
}
return Flags;
@@ -1441,8 +1457,9 @@ static uint64_t rotateSign(int64_t I) {
void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
SmallVectorImpl<uint64_t> &Record,
unsigned Abbrev) {
- Record.push_back(N->isDistinct());
- Record.push_back(N->getCount());
+ const uint64_t Version = 1 << 1;
+ Record.push_back((uint64_t)N->isDistinct() | Version);
+ Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode()));
Record.push_back(rotateSign(N->getLowerBound()));
Stream.EmitRecord(bitc::METADATA_SUBRANGE, Record, Abbrev);
@@ -1452,7 +1469,7 @@ void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
SmallVectorImpl<uint64_t> &Record,
unsigned Abbrev) {
- Record.push_back(N->isDistinct());
+ Record.push_back((N->isUnsigned() << 1) | N->isDistinct());
Record.push_back(rotateSign(N->getValue()));
Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
@@ -1521,6 +1538,7 @@ void ModuleBitcodeWriter::writeDICompositeType(
Record.push_back(VE.getMetadataOrNullID(N->getVTableHolder()));
Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get()));
Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier()));
+ Record.push_back(VE.getMetadataOrNullID(N->getDiscriminator()));
Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev);
Record.clear();
@@ -1545,8 +1563,18 @@ void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
Record.push_back(N->isDistinct());
Record.push_back(VE.getMetadataOrNullID(N->getRawFilename()));
Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory()));
- Record.push_back(N->getChecksumKind());
- Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum()));
+ if (N->getRawChecksum()) {
+ Record.push_back(N->getRawChecksum()->Kind);
+ Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum()->Value));
+ } else {
+ // Maintain backwards compatibility with the old internal representation of
+ // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
+ Record.push_back(0);
+ Record.push_back(VE.getMetadataOrNullID(nullptr));
+ }
+ auto Source = N->getRawSource();
+ if (Source)
+ Record.push_back(VE.getMetadataOrNullID(*Source));
Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev);
Record.clear();
@@ -1602,7 +1630,7 @@ void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
Record.push_back(VE.getMetadataOrNullID(N->getRawUnit()));
Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get()));
Record.push_back(VE.getMetadataOrNullID(N->getDeclaration()));
- Record.push_back(VE.getMetadataOrNullID(N->getVariables().get()));
+ Record.push_back(VE.getMetadataOrNullID(N->getRetainedNodes().get()));
Record.push_back(N->getThisAdjustment());
Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get()));
@@ -1759,6 +1787,19 @@ void ModuleBitcodeWriter::writeDILocalVariable(
Record.clear();
}
+void ModuleBitcodeWriter::writeDILabel(
+ const DILabel *N, SmallVectorImpl<uint64_t> &Record,
+ unsigned Abbrev) {
+ Record.push_back((uint64_t)N->isDistinct());
+ Record.push_back(VE.getMetadataOrNullID(N->getScope()));
+ Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
+ Record.push_back(VE.getMetadataOrNullID(N->getFile()));
+ Record.push_back(N->getLine());
+
+ Stream.EmitRecord(bitc::METADATA_LABEL, Record, Abbrev);
+ Record.clear();
+}
+
void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
SmallVectorImpl<uint64_t> &Record,
unsigned Abbrev) {
@@ -3183,7 +3224,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); // flags
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
FUNCTION_INST_BINOP_FLAGS_ABBREV)
llvm_unreachable("Unexpected abbrev ordering!");
@@ -3312,10 +3353,14 @@ void IndexBitcodeWriter::writeModStrings() {
/// Write the function type metadata related records that need to appear before
/// a function summary entry (whether per-module or combined).
-static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
- FunctionSummary *FS) {
- if (!FS->type_tests().empty())
+static void writeFunctionTypeMetadataRecords(
+ BitstreamWriter &Stream, FunctionSummary *FS,
+ std::set<GlobalValue::GUID> &ReferencedTypeIds) {
+ if (!FS->type_tests().empty()) {
Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests());
+ for (auto &TT : FS->type_tests())
+ ReferencedTypeIds.insert(TT);
+ }
SmallVector<uint64_t, 64> Record;
@@ -3327,6 +3372,7 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
for (auto &VF : VFs) {
Record.push_back(VF.GUID);
Record.push_back(VF.Offset);
+ ReferencedTypeIds.insert(VF.GUID);
}
Stream.EmitRecord(Ty, Record);
};
@@ -3341,6 +3387,7 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
for (auto &VC : VCs) {
Record.clear();
Record.push_back(VC.VFunc.GUID);
+ ReferencedTypeIds.insert(VC.VFunc.GUID);
Record.push_back(VC.VFunc.Offset);
Record.insert(Record.end(), VC.Args.begin(), VC.Args.end());
Stream.EmitRecord(Ty, Record);
@@ -3353,6 +3400,51 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
FS->type_checked_load_const_vcalls());
}
+static void writeWholeProgramDevirtResolutionByArg(
+ SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
+ const WholeProgramDevirtResolution::ByArg &ByArg) {
+ NameVals.push_back(args.size());
+ NameVals.insert(NameVals.end(), args.begin(), args.end());
+
+ NameVals.push_back(ByArg.TheKind);
+ NameVals.push_back(ByArg.Info);
+ NameVals.push_back(ByArg.Byte);
+ NameVals.push_back(ByArg.Bit);
+}
+
+static void writeWholeProgramDevirtResolution(
+ SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
+ uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
+ NameVals.push_back(Id);
+
+ NameVals.push_back(Wpd.TheKind);
+ NameVals.push_back(StrtabBuilder.add(Wpd.SingleImplName));
+ NameVals.push_back(Wpd.SingleImplName.size());
+
+ NameVals.push_back(Wpd.ResByArg.size());
+ for (auto &A : Wpd.ResByArg)
+ writeWholeProgramDevirtResolutionByArg(NameVals, A.first, A.second);
+}
+
+static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
+ StringTableBuilder &StrtabBuilder,
+ const std::string &Id,
+ const TypeIdSummary &Summary) {
+ NameVals.push_back(StrtabBuilder.add(Id));
+ NameVals.push_back(Id.size());
+
+ NameVals.push_back(Summary.TTRes.TheKind);
+ NameVals.push_back(Summary.TTRes.SizeM1BitWidth);
+ NameVals.push_back(Summary.TTRes.AlignLog2);
+ NameVals.push_back(Summary.TTRes.SizeM1);
+ NameVals.push_back(Summary.TTRes.BitMask);
+ NameVals.push_back(Summary.TTRes.InlineBits);
+
+ for (auto &W : Summary.WPDRes)
+ writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, W.first,
+ W.second);
+}
+
// Helper to emit a single function summary record.
void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
@@ -3361,7 +3453,8 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
NameVals.push_back(ValueID);
FunctionSummary *FS = cast<FunctionSummary>(Summary);
- writeFunctionTypeMetadataRecords(Stream, FS);
+ std::set<GlobalValue::GUID> ReferencedTypeIds;
+ writeFunctionTypeMetadataRecords(Stream, FS, ReferencedTypeIds);
NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
NameVals.push_back(FS->instCount());
@@ -3371,16 +3464,21 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
for (auto &RI : FS->refs())
NameVals.push_back(VE.getValueID(RI.getValue()));
- bool HasProfileData = F.hasProfileData();
+ bool HasProfileData =
+ F.hasProfileData() || ForceSummaryEdgesCold != FunctionSummary::FSHT_None;
for (auto &ECI : FS->calls()) {
NameVals.push_back(getValueId(ECI.first));
if (HasProfileData)
NameVals.push_back(static_cast<uint8_t>(ECI.second.Hotness));
+ else if (WriteRelBFToSummary)
+ NameVals.push_back(ECI.second.RelBlockFreq);
}
unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
unsigned Code =
- (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE);
+ (HasProfileData ? bitc::FS_PERMODULE_PROFILE
+ : (WriteRelBFToSummary ? bitc::FS_PERMODULE_RELBF
+ : bitc::FS_PERMODULE));
// Emit the finished record.
Stream.EmitRecord(Code, NameVals, FSAbbrev);
@@ -3392,7 +3490,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
void ModuleBitcodeWriterBase::writeModuleLevelReferences(
const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
unsigned FSModRefsAbbrev) {
- auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName()));
+ auto VI = Index->getValueInfo(V.getGUID());
if (!VI || VI.getSummaryList().empty()) {
// Only declarations should not have a summary (a declaration might however
// have a summary if the def was in module level asm).
@@ -3409,7 +3507,7 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences(
NameVals.push_back(VE.getValueID(RI.getValue()));
// Sort the refs for determinism output, the vector returned by FS->refs() has
// been initialized from a DenseSet.
- std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end());
+ llvm::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end());
Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
FSModRefsAbbrev);
@@ -3446,31 +3544,34 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
ArrayRef<uint64_t>{GVI.second, GVI.first});
}
- // Abbrev for FS_PERMODULE.
+ // Abbrev for FS_PERMODULE_PROFILE.
auto Abbv = std::make_shared<BitCodeAbbrev>();
- Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
- // numrefs x valueid, n x (valueid)
+ // numrefs x valueid, n x (valueid, hotness)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
- unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
+ unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
- // Abbrev for FS_PERMODULE_PROFILE.
+ // Abbrev for FS_PERMODULE or FS_PERMODULE_RELBF.
Abbv = std::make_shared<BitCodeAbbrev>();
- Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
+ if (WriteRelBFToSummary)
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF));
+ else
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
- // numrefs x valueid, n x (valueid, hotness)
+ // numrefs x valueid, n x (valueid [, rel_block_freq])
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
- unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
+ unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
// Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
Abbv = std::make_shared<BitCodeAbbrev>();
@@ -3498,7 +3599,7 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
if (!F.hasName())
report_fatal_error("Unexpected anonymous function when writing summary");
- ValueInfo VI = Index->getValueInfo(GlobalValue::getGUID(F.getName()));
+ ValueInfo VI = Index->getValueInfo(F.getGUID());
if (!VI || VI.getSummaryList().empty()) {
// Only declarations should not have a summary (a declaration might
// however have a summary if the def was in module level asm).
@@ -3539,6 +3640,14 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION});
+ // Write the index flags.
+ uint64_t Flags = 0;
+ if (Index.withGlobalValueDeadStripping())
+ Flags |= 0x1;
+ if (Index.skipModuleByDistributedBackend())
+ Flags |= 0x2;
+ Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
+
for (const auto &GVI : valueIds()) {
Stream.EmitRecord(bitc::FS_VALUE_GUID,
ArrayRef<uint64_t>{GVI.second, GVI.first});
@@ -3600,6 +3709,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
SmallVector<uint64_t, 64> NameVals;
+ // Set that will be populated during call to writeFunctionTypeMetadataRecords
+ // with the type ids referenced by this index file.
+ std::set<GlobalValue::GUID> ReferencedTypeIds;
+
// For local linkage, we also emit the original name separately
// immediately after the record.
auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
@@ -3651,7 +3764,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
}
auto *FS = cast<FunctionSummary>(S);
- writeFunctionTypeMetadataRecords(Stream, FS);
+ writeFunctionTypeMetadataRecords(Stream, FS, ReferencedTypeIds);
NameVals.push_back(*ValueId);
NameVals.push_back(Index.getModuleId(FS->modulePath()));
@@ -3673,7 +3786,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
bool HasProfileData = false;
for (auto &EI : FS->calls()) {
- HasProfileData |= EI.second.Hotness != CalleeInfo::HotnessType::Unknown;
+ HasProfileData |=
+ EI.second.getHotness() != CalleeInfo::HotnessType::Unknown;
if (HasProfileData)
break;
}
@@ -3757,6 +3871,17 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
NameVals.clear();
}
+ if (!Index.typeIds().empty()) {
+ for (auto &S : Index.typeIds()) {
+ // Skip if not referenced in any GV summary within this index file.
+ if (!ReferencedTypeIds.count(GlobalValue::getGUID(S.first)))
+ continue;
+ writeTypeIdSummaryRecord(NameVals, StrtabBuilder, S.first, S.second);
+ Stream.EmitRecord(bitc::FS_TYPE_ID, NameVals);
+ NameVals.clear();
+ }
+ }
+
Stream.ExitBlock();
}
@@ -4012,7 +4137,7 @@ void BitcodeWriter::copyStrtab(StringRef Strtab) {
WroteStrtab = true;
}
-void BitcodeWriter::writeModule(const Module *M,
+void BitcodeWriter::writeModule(const Module &M,
bool ShouldPreserveUseListOrder,
const ModuleSummaryIndex *Index,
bool GenerateHash, ModuleHash *ModHash) {
@@ -4022,8 +4147,8 @@ void BitcodeWriter::writeModule(const Module *M,
// Modules in case it needs to materialize metadata. But the bitcode writer
// requires that the module is materialized, so we can cast to non-const here,
// after checking that it is in fact materialized.
- assert(M->isMaterialized());
- Mods.push_back(const_cast<Module *>(M));
+ assert(M.isMaterialized());
+ Mods.push_back(const_cast<Module *>(&M));
ModuleBitcodeWriter ModuleWriter(M, Buffer, StrtabBuilder, *Stream,
ShouldPreserveUseListOrder, Index,
@@ -4039,9 +4164,8 @@ void BitcodeWriter::writeIndex(
IndexWriter.write();
}
-/// WriteBitcodeToFile - Write the specified module to the specified output
-/// stream.
-void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
+/// Write the specified module to the specified output stream.
+void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
bool ShouldPreserveUseListOrder,
const ModuleSummaryIndex *Index,
bool GenerateHash, ModuleHash *ModHash) {
@@ -4050,7 +4174,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
// If this is darwin or another generic macho target, reserve space for the
// header.
- Triple TT(M->getTargetTriple());
+ Triple TT(M.getTargetTriple());
if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0);
@@ -4107,7 +4231,7 @@ class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
const ModuleHash *ModHash;
public:
- ThinLinkBitcodeWriter(const Module *M, StringTableBuilder &StrtabBuilder,
+ ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
BitstreamWriter &Stream,
const ModuleSummaryIndex &Index,
const ModuleHash &ModHash)
@@ -4225,7 +4349,7 @@ void ThinLinkBitcodeWriter::write() {
Stream.ExitBlock();
}
-void BitcodeWriter::writeThinLinkBitcode(const Module *M,
+void BitcodeWriter::writeThinLinkBitcode(const Module &M,
const ModuleSummaryIndex &Index,
const ModuleHash &ModHash) {
assert(!WroteStrtab);
@@ -4234,8 +4358,8 @@ void BitcodeWriter::writeThinLinkBitcode(const Module *M,
// Modules in case it needs to materialize metadata. But the bitcode writer
// requires that the module is materialized, so we can cast to non-const here,
// after checking that it is in fact materialized.
- assert(M->isMaterialized());
- Mods.push_back(const_cast<Module *>(M));
+ assert(M.isMaterialized());
+ Mods.push_back(const_cast<Module *>(&M));
ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
ModHash);
@@ -4245,7 +4369,7 @@ void BitcodeWriter::writeThinLinkBitcode(const Module *M,
// Write the specified thin link bitcode file to the given raw output stream,
// where it will be written in a new bitcode block. This is used when
// writing the per-module index file for ThinLTO.
-void llvm::WriteThinLinkBitcodeToFile(const Module *M, raw_ostream &Out,
+void llvm::WriteThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
const ModuleSummaryIndex &Index,
const ModuleHash &ModHash) {
SmallVector<char, 0> Buffer;
diff --git a/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/lib/Bitcode/Writer/BitcodeWriterPass.cpp
index 80cab762a68c..41212e575f8e 100644
--- a/lib/Bitcode/Writer/BitcodeWriterPass.cpp
+++ b/lib/Bitcode/Writer/BitcodeWriterPass.cpp
@@ -23,7 +23,7 @@ PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
const ModuleSummaryIndex *Index =
EmitSummaryIndex ? &(AM.getResult<ModuleSummaryIndexAnalysis>(M))
: nullptr;
- WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash);
+ WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash);
return PreservedAnalyses::all();
}
@@ -55,7 +55,7 @@ namespace {
EmitSummaryIndex
? &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex())
: nullptr;
- WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index,
+ WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, Index,
EmitModuleHash);
return false;
}
@@ -80,3 +80,7 @@ ModulePass *llvm::createBitcodeWriterPass(raw_ostream &Str,
return new WriteBitcodePass(Str, ShouldPreserveUseListOrder,
EmitSummaryIndex, EmitModuleHash);
}
+
+bool llvm::isBitcodeWriterPass(Pass *P) {
+ return P->getPassID() == (llvm::AnalysisID)&WriteBitcodePass::ID;
+}
diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp
index d99befcdaeae..d473741e8ceb 100644
--- a/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -14,6 +14,7 @@
#include "ValueEnumerator.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
@@ -183,7 +184,7 @@ static void predictValueUseListOrderImpl(const Value *V, const Function *F,
return;
bool IsGlobalValue = OM.isGlobalValue(ID);
- std::sort(List.begin(), List.end(), [&](const Entry &L, const Entry &R) {
+ llvm::sort(List.begin(), List.end(), [&](const Entry &L, const Entry &R) {
const Use *LU = L.first;
const Use *RU = R.first;
if (LU == RU)
@@ -488,7 +489,7 @@ void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map,
V->print(errs());
errs() << '\n';
- OS << " Uses(" << std::distance(V->use_begin(),V->use_end()) << "):";
+ OS << " Uses(" << V->getNumUses() << "):";
for (const Use &U : V->uses()) {
if (&U != &*V->use_begin())
OS << ",";
@@ -744,7 +745,7 @@ void ValueEnumerator::organizeMetadata() {
// and then sort by the original/current ID. Since the IDs are guaranteed to
// be unique, the result of std::sort will be deterministic. There's no need
// for std::stable_sort.
- std::sort(Order.begin(), Order.end(), [this](MDIndex LHS, MDIndex RHS) {
+ llvm::sort(Order.begin(), Order.end(), [this](MDIndex LHS, MDIndex RHS) {
return std::make_tuple(LHS.F, getMetadataTypeOrder(LHS.get(MDs)), LHS.ID) <
std::make_tuple(RHS.F, getMetadataTypeOrder(RHS.get(MDs)), RHS.ID);
});