aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2023-04-14 21:41:27 +0000
committerDimitry Andric <dim@FreeBSD.org>2023-06-22 18:20:56 +0000
commitbdd1243df58e60e85101c09001d9812a789b6bc4 (patch)
treea1ce621c7301dd47ba2ddc3b8eaa63b441389481 /contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
parent781624ca2d054430052c828ba8d2c2eaf2d733e7 (diff)
parente3b557809604d036af6e00c60f012c2025b59a5e (diff)
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp583
1 files changed, 424 insertions, 159 deletions
diff --git a/contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 1943b5db94c3..f014521264c1 100644
--- a/contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/contrib/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -13,7 +13,6 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
@@ -71,6 +70,7 @@
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/ModRef.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
@@ -79,6 +79,7 @@
#include <deque>
#include <map>
#include <memory>
+#include <optional>
#include <set>
#include <string>
#include <system_error>
@@ -546,13 +547,13 @@ public:
static bool classof(const Value *V) { return V->getValueID() == SubclassID; }
ArrayRef<unsigned> getOperandIDs() const {
- return makeArrayRef(getTrailingObjects<unsigned>(), NumOperands);
+ return ArrayRef(getTrailingObjects<unsigned>(), NumOperands);
}
- Optional<unsigned> getInRangeIndex() const {
+ std::optional<unsigned> getInRangeIndex() const {
assert(Opcode == Instruction::GetElementPtr);
if (Extra == (unsigned)-1)
- return None;
+ return std::nullopt;
return Extra;
}
@@ -592,7 +593,7 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer {
/// ValueList must be destroyed before Alloc.
BumpPtrAllocator Alloc;
BitcodeReaderValueList ValueList;
- Optional<MetadataLoader> MDLoader;
+ std::optional<MetadataLoader> MDLoader;
std::vector<Comdat *> ComdatList;
DenseSet<GlobalObject *> ImplicitComdatObjects;
SmallVector<Instruction *, 64> InstructionList;
@@ -627,8 +628,6 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer {
// stored here with their replacement function.
using UpdatedIntrinsicMap = DenseMap<Function *, Function *>;
UpdatedIntrinsicMap UpgradedIntrinsics;
- // Intrinsics which were remangled because of types rename
- UpdatedIntrinsicMap RemangledIntrinsics;
// Several operations happen after the module header has been read, but
// before function bodies are processed. This keeps track of whether
@@ -675,6 +674,8 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer {
std::vector<std::string> BundleTags;
SmallVector<SyncScope::ID, 8> SSIDs;
+ std::optional<ValueTypeCallbackTy> ValueTypeCallback;
+
public:
BitcodeReader(BitstreamCursor Stream, StringRef Strtab,
StringRef ProducerIdentification, LLVMContext &Context);
@@ -687,9 +688,8 @@ public:
/// Main interface to parsing a bitcode buffer.
/// \returns true if an error occurred.
- Error parseBitcodeInto(
- Module *M, bool ShouldLazyLoadMetadata, bool IsImporting,
- DataLayoutCallbackTy DataLayoutCallback);
+ Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata,
+ bool IsImporting, ParserCallbacks Callbacks = {});
static uint64_t decodeSignRotatedValue(uint64_t V);
@@ -710,6 +710,7 @@ private:
unsigned getContainedTypeID(unsigned ID, unsigned Idx = 0);
unsigned getVirtualTypeID(Type *Ty, ArrayRef<unsigned> ContainedTypeIDs = {});
+ void callValueTypeCallback(Value *F, unsigned TypeID);
Expected<Value *> materializeValue(unsigned ValID, BasicBlock *InsertBB);
Expected<Constant *> getValueForInitializer(unsigned ID);
@@ -820,9 +821,8 @@ private:
/// a corresponding error code.
Error parseAlignmentValue(uint64_t Exponent, MaybeAlign &Alignment);
Error parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind);
- Error parseModule(
- uint64_t ResumeBit, bool ShouldLazyLoadMetadata = false,
- DataLayoutCallbackTy DataLayoutCallback = [](StringRef) { return None; });
+ Error parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata = false,
+ ParserCallbacks Callbacks = {});
Error parseComdatRecord(ArrayRef<uint64_t> Record);
Error parseGlobalVarRecord(ArrayRef<uint64_t> Record);
@@ -884,8 +884,10 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase {
// they are recorded in the summary index being built.
// We save a GUID which refers to the same global as the ValueInfo, but
// ignoring the linkage, i.e. for values other than local linkage they are
- // identical.
- DenseMap<unsigned, std::pair<ValueInfo, GlobalValue::GUID>>
+ // identical (this is the second tuple member).
+ // The third tuple member is the real GUID of the ValueInfo.
+ DenseMap<unsigned,
+ std::tuple<ValueInfo, GlobalValue::GUID, GlobalValue::GUID>>
ValueIdToValueInfoMap;
/// Map populated during module path string table parsing, from the
@@ -905,10 +907,19 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase {
/// this module by the client.
unsigned ModuleId;
+ /// Callback to ask whether a symbol is the prevailing copy when invoked
+ /// during combined index building.
+ std::function<bool(GlobalValue::GUID)> IsPrevailing;
+
+ /// Saves the stack ids from the STACK_IDS record to consult when adding stack
+ /// ids from the lists in the callsite and alloc entries to the index.
+ std::vector<uint64_t> StackIds;
+
public:
- ModuleSummaryIndexBitcodeReader(BitstreamCursor Stream, StringRef Strtab,
- ModuleSummaryIndex &TheIndex,
- StringRef ModulePath, unsigned ModuleId);
+ ModuleSummaryIndexBitcodeReader(
+ BitstreamCursor Stream, StringRef Strtab, ModuleSummaryIndex &TheIndex,
+ StringRef ModulePath, unsigned ModuleId,
+ std::function<bool(GlobalValue::GUID)> IsPrevailing = nullptr);
Error parseModule();
@@ -932,7 +943,8 @@ private:
std::vector<FunctionSummary::ParamAccess>
parseParamAccesses(ArrayRef<uint64_t> Record);
- std::pair<ValueInfo, GlobalValue::GUID>
+ template <bool AllowNullValueInfo = false>
+ std::tuple<ValueInfo, GlobalValue::GUID, GlobalValue::GUID>
getValueInfoFromValueId(unsigned ValueId);
void addThisModule();
@@ -1244,6 +1256,10 @@ static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) {
case bitc::RMW_FSUB: return AtomicRMWInst::FSub;
case bitc::RMW_FMAX: return AtomicRMWInst::FMax;
case bitc::RMW_FMIN: return AtomicRMWInst::FMin;
+ case bitc::RMW_UINC_WRAP:
+ return AtomicRMWInst::UIncWrap;
+ case bitc::RMW_UDEC_WRAP:
+ return AtomicRMWInst::UDecWrap;
}
}
@@ -1298,6 +1314,9 @@ static FastMathFlags getDecodedFastMathFlags(unsigned Val) {
}
static void upgradeDLLImportExportLinkage(GlobalValue *GV, unsigned Val) {
+ // A GlobalValue with local linkage cannot have a DLL storage class.
+ if (GV->hasLocalLinkage())
+ return;
switch (Val) {
case 5: GV->setDLLStorageClass(GlobalValue::DLLImportStorageClass); break;
case 6: GV->setDLLStorageClass(GlobalValue::DLLExportStorageClass); break;
@@ -1385,10 +1404,15 @@ static bool isConstExprSupported(uint8_t Opcode) {
if (Opcode >= BitcodeConstant::FirstSpecialOpcode)
return true;
+ // If -expand-constant-exprs is set, we want to consider all expressions
+ // as unsupported.
+ if (ExpandConstantExprs)
+ return false;
+
if (Instruction::isBinaryOp(Opcode))
return ConstantExpr::isSupportedBinOp(Opcode);
- return !ExpandConstantExprs;
+ return Opcode != Instruction::FNeg;
}
Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
@@ -1449,8 +1473,6 @@ Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
C = UpgradeBitCastExpr(BC->Opcode, ConstOps[0], BC->getType());
if (!C)
C = ConstantExpr::getCast(BC->Opcode, ConstOps[0], BC->getType());
- } else if (Instruction::isUnaryOp(BC->Opcode)) {
- C = ConstantExpr::get(BC->Opcode, ConstOps[0], BC->Flags);
} else if (Instruction::isBinaryOp(BC->Opcode)) {
C = ConstantExpr::get(BC->Opcode, ConstOps[0], ConstOps[1], BC->Flags);
} else {
@@ -1518,9 +1540,9 @@ Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
C = ConstantExpr::getCompare(BC->Flags, ConstOps[0], ConstOps[1]);
break;
case Instruction::GetElementPtr:
- C = ConstantExpr::getGetElementPtr(
- BC->SrcElemTy, ConstOps[0], makeArrayRef(ConstOps).drop_front(),
- BC->Flags, BC->getInRangeIndex());
+ C = ConstantExpr::getGetElementPtr(BC->SrcElemTy, ConstOps[0],
+ ArrayRef(ConstOps).drop_front(),
+ BC->Flags, BC->getInRangeIndex());
break;
case Instruction::Select:
C = ConstantExpr::getSelect(ConstOps[0], ConstOps[1], ConstOps[2]);
@@ -1577,8 +1599,6 @@ Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
I->setIsExact();
} else {
switch (BC->Opcode) {
- case BitcodeConstant::ConstantStructOpcode:
- case BitcodeConstant::ConstantArrayOpcode:
case BitcodeConstant::ConstantVectorOpcode: {
Type *IdxTy = Type::getInt32Ty(BC->getContext());
Value *V = PoisonValue::get(BC->getType());
@@ -1590,6 +1610,15 @@ Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
I = cast<Instruction>(V);
break;
}
+ case BitcodeConstant::ConstantStructOpcode:
+ case BitcodeConstant::ConstantArrayOpcode: {
+ Value *V = PoisonValue::get(BC->getType());
+ for (auto Pair : enumerate(Ops))
+ V = InsertValueInst::Create(V, Pair.value(), Pair.index(),
+ "constexpr.ins", InsertBB);
+ I = cast<Instruction>(V);
+ break;
+ }
case Instruction::ICmp:
case Instruction::FCmp:
I = CmpInst::Create((Instruction::OtherOps)BC->Opcode,
@@ -1598,8 +1627,8 @@ Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
break;
case Instruction::GetElementPtr:
I = GetElementPtrInst::Create(BC->SrcElemTy, Ops[0],
- makeArrayRef(Ops).drop_front(),
- "constexpr", InsertBB);
+ ArrayRef(Ops).drop_front(), "constexpr",
+ InsertBB);
if (BC->Flags)
cast<GetElementPtrInst>(I)->setIsInBounds();
break;
@@ -1706,8 +1735,8 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) {
case Attribute::Convergent: return 1ULL << 46;
case Attribute::SafeStack: return 1ULL << 47;
case Attribute::NoRecurse: return 1ULL << 48;
- case Attribute::InaccessibleMemOnly: return 1ULL << 49;
- case Attribute::InaccessibleMemOrArgMemOnly: return 1ULL << 50;
+ // 1ULL << 49 is InaccessibleMemOnly, which is upgraded separately.
+ // 1ULL << 50 is InaccessibleMemOrArgMemOnly, which is upgraded separately.
case Attribute::SwiftSelf: return 1ULL << 51;
case Attribute::SwiftError: return 1ULL << 52;
case Attribute::WriteOnly: return 1ULL << 53;
@@ -1755,7 +1784,8 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) {
/// been decoded from the given integer. This function must stay in sync with
/// 'encodeLLVMAttributesForBitcode'.
static void decodeLLVMAttributesForBitcode(AttrBuilder &B,
- uint64_t EncodedAttrs) {
+ uint64_t EncodedAttrs,
+ uint64_t AttrIdx) {
// The alignment is stored as a 16-bit raw value from bits 31--16. We shift
// the bits above 31 down by 11 bits.
unsigned Alignment = (EncodedAttrs & (0xffffULL << 16)) >> 16;
@@ -1764,8 +1794,43 @@ static void decodeLLVMAttributesForBitcode(AttrBuilder &B,
if (Alignment)
B.addAlignmentAttr(Alignment);
- addRawAttributeValue(B, ((EncodedAttrs & (0xfffffULL << 32)) >> 11) |
- (EncodedAttrs & 0xffff));
+
+ uint64_t Attrs = ((EncodedAttrs & (0xfffffULL << 32)) >> 11) |
+ (EncodedAttrs & 0xffff);
+
+ if (AttrIdx == AttributeList::FunctionIndex) {
+ // Upgrade old memory attributes.
+ MemoryEffects ME = MemoryEffects::unknown();
+ if (Attrs & (1ULL << 9)) {
+ // ReadNone
+ Attrs &= ~(1ULL << 9);
+ ME &= MemoryEffects::none();
+ }
+ if (Attrs & (1ULL << 10)) {
+ // ReadOnly
+ Attrs &= ~(1ULL << 10);
+ ME &= MemoryEffects::readOnly();
+ }
+ if (Attrs & (1ULL << 49)) {
+ // InaccessibleMemOnly
+ Attrs &= ~(1ULL << 49);
+ ME &= MemoryEffects::inaccessibleMemOnly();
+ }
+ if (Attrs & (1ULL << 50)) {
+ // InaccessibleMemOrArgMemOnly
+ Attrs &= ~(1ULL << 50);
+ ME &= MemoryEffects::inaccessibleOrArgMemOnly();
+ }
+ if (Attrs & (1ULL << 53)) {
+ // WriteOnly
+ Attrs &= ~(1ULL << 53);
+ ME &= MemoryEffects::writeOnly();
+ }
+ if (ME != MemoryEffects::unknown())
+ B.addMemoryAttr(ME);
+ }
+
+ addRawAttributeValue(B, Attrs);
}
Error BitcodeReader::parseAttributeBlock() {
@@ -1812,7 +1877,7 @@ Error BitcodeReader::parseAttributeBlock() {
for (unsigned i = 0, e = Record.size(); i != e; i += 2) {
AttrBuilder B(Context);
- decodeLLVMAttributesForBitcode(B, Record[i+1]);
+ decodeLLVMAttributesForBitcode(B, Record[i+1], Record[i]);
Attrs.push_back(AttributeList::get(Context, Record[i], B));
}
@@ -1839,8 +1904,6 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::Alignment;
case bitc::ATTR_KIND_ALWAYS_INLINE:
return Attribute::AlwaysInline;
- case bitc::ATTR_KIND_ARGMEMONLY:
- return Attribute::ArgMemOnly;
case bitc::ATTR_KIND_BUILTIN:
return Attribute::Builtin;
case bitc::ATTR_KIND_BY_VAL:
@@ -1857,16 +1920,14 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::ElementType;
case bitc::ATTR_KIND_FNRETTHUNK_EXTERN:
return Attribute::FnRetThunkExtern;
- case bitc::ATTR_KIND_INACCESSIBLEMEM_ONLY:
- return Attribute::InaccessibleMemOnly;
- case bitc::ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY:
- return Attribute::InaccessibleMemOrArgMemOnly;
case bitc::ATTR_KIND_INLINE_HINT:
return Attribute::InlineHint;
case bitc::ATTR_KIND_IN_REG:
return Attribute::InReg;
case bitc::ATTR_KIND_JUMP_TABLE:
return Attribute::JumpTable;
+ case bitc::ATTR_KIND_MEMORY:
+ return Attribute::Memory;
case bitc::ATTR_KIND_MIN_SIZE:
return Attribute::MinSize;
case bitc::ATTR_KIND_NAKED:
@@ -1919,6 +1980,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::NoCfCheck;
case bitc::ATTR_KIND_NO_PROFILE:
return Attribute::NoProfile;
+ case bitc::ATTR_KIND_SKIP_PROFILE:
+ return Attribute::SkipProfile;
case bitc::ATTR_KIND_NO_UNWIND:
return Attribute::NoUnwind;
case bitc::ATTR_KIND_NO_SANITIZE_BOUNDS:
@@ -2023,6 +2086,31 @@ Error BitcodeReader::parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind) {
return Error::success();
}
+static bool upgradeOldMemoryAttribute(MemoryEffects &ME, uint64_t EncodedKind) {
+ switch (EncodedKind) {
+ case bitc::ATTR_KIND_READ_NONE:
+ ME &= MemoryEffects::none();
+ return true;
+ case bitc::ATTR_KIND_READ_ONLY:
+ ME &= MemoryEffects::readOnly();
+ return true;
+ case bitc::ATTR_KIND_WRITEONLY:
+ ME &= MemoryEffects::writeOnly();
+ return true;
+ case bitc::ATTR_KIND_ARGMEMONLY:
+ ME &= MemoryEffects::argMemOnly();
+ return true;
+ case bitc::ATTR_KIND_INACCESSIBLEMEM_ONLY:
+ ME &= MemoryEffects::inaccessibleMemOnly();
+ return true;
+ case bitc::ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY:
+ ME &= MemoryEffects::inaccessibleOrArgMemOnly();
+ return true;
+ default:
+ return false;
+ }
+}
+
Error BitcodeReader::parseAttributeGroupBlock() {
if (Error Err = Stream.EnterSubBlock(bitc::PARAMATTR_GROUP_BLOCK_ID))
return Err;
@@ -2066,10 +2154,16 @@ Error BitcodeReader::parseAttributeGroupBlock() {
uint64_t Idx = Record[1]; // Index of the object this attribute refers to.
AttrBuilder B(Context);
+ MemoryEffects ME = MemoryEffects::unknown();
for (unsigned i = 2, e = Record.size(); i != e; ++i) {
if (Record[i] == 0) { // Enum attribute
Attribute::AttrKind Kind;
- if (Error Err = parseAttrKind(Record[++i], &Kind))
+ uint64_t EncodedKind = Record[++i];
+ if (Idx == AttributeList::FunctionIndex &&
+ upgradeOldMemoryAttribute(ME, EncodedKind))
+ continue;
+
+ if (Error Err = parseAttrKind(EncodedKind, &Kind))
return Err;
// Upgrade old-style byval attribute to one with a type, even if it's
@@ -2109,6 +2203,8 @@ Error BitcodeReader::parseAttributeGroupBlock() {
B.addUWTableAttr(UWTableKind(Record[++i]));
else if (Kind == Attribute::AllocKind)
B.addAllocKindAttr(static_cast<AllocFnKind>(Record[++i]));
+ else if (Kind == Attribute::Memory)
+ B.addMemoryAttr(MemoryEffects::createFromIntValue(Record[++i]));
} else if (Record[i] == 3 || Record[i] == 4) { // String attribute
bool HasValue = (Record[i++] == 4);
SmallString<64> KindStr;
@@ -2141,6 +2237,9 @@ Error BitcodeReader::parseAttributeGroupBlock() {
}
}
+ if (ME != MemoryEffects::unknown())
+ B.addMemoryAttr(ME);
+
UpgradeAttributes(B);
MAttributeGroups[GrpID] = AttributeList::get(Context, Idx, B);
break;
@@ -2263,8 +2362,6 @@ Error BitcodeReader::parseTypeTableBody() {
if (!ResultTy ||
!PointerType::isValidElementType(ResultTy))
return error("Invalid type");
- if (LLVM_UNLIKELY(!Context.hasSetOpaquePointersValue()))
- Context.setOpaquePointers(false);
ContainedIDs.push_back(Record[0]);
ResultTy = PointerType::get(ResultTy, AddressSpace);
break;
@@ -2272,9 +2369,7 @@ Error BitcodeReader::parseTypeTableBody() {
case bitc::TYPE_CODE_OPAQUE_POINTER: { // OPAQUE_POINTER: [addrspace]
if (Record.size() != 1)
return error("Invalid opaque pointer record");
- if (LLVM_UNLIKELY(!Context.hasSetOpaquePointersValue())) {
- Context.setOpaquePointers(true);
- } else if (Context.supportsTypedPointers())
+ if (Context.supportsTypedPointers())
return error(
"Opaque pointers are only supported in -opaque-pointers mode");
unsigned AddressSpace = Record[0];
@@ -2394,6 +2489,35 @@ Error BitcodeReader::parseTypeTableBody() {
ResultTy = Res;
break;
}
+ case bitc::TYPE_CODE_TARGET_TYPE: { // TARGET_TYPE: [NumTy, Tys..., Ints...]
+ if (Record.size() < 1)
+ return error("Invalid target extension type record");
+
+ if (NumRecords >= TypeList.size())
+ return error("Invalid TYPE table");
+
+ if (Record[0] >= Record.size())
+ return error("Too many type parameters");
+
+ unsigned NumTys = Record[0];
+ SmallVector<Type *, 4> TypeParams;
+ SmallVector<unsigned, 8> IntParams;
+ for (unsigned i = 0; i < NumTys; i++) {
+ if (Type *T = getTypeByID(Record[i + 1]))
+ TypeParams.push_back(T);
+ else
+ return error("Invalid type");
+ }
+
+ for (unsigned i = NumTys + 1, e = Record.size(); i < e; i++) {
+ if (Record[i] > UINT_MAX)
+ return error("Integer parameter too large");
+ IntParams.push_back(Record[i]);
+ }
+ ResultTy = TargetExtType::get(Context, TypeName, TypeParams, IntParams);
+ TypeName.clear();
+ break;
+ }
case bitc::TYPE_CODE_ARRAY: // ARRAY: [numelts, eltty]
if (Record.size() < 2)
return error("Invalid array type record");
@@ -2778,9 +2902,9 @@ Error BitcodeReader::resolveGlobalAndIndirectSymbolInits() {
} else if (auto *GI = dyn_cast<GlobalIFunc>(GV)) {
Type *ResolverFTy =
GlobalIFunc::getResolverFunctionType(GI->getValueType());
- // Transparently fix up the type for compatiblity with older bitcode
- GI->setResolver(
- ConstantExpr::getBitCast(C, ResolverFTy->getPointerTo()));
+ // Transparently fix up the type for compatibility with older bitcode
+ GI->setResolver(ConstantExpr::getBitCast(
+ C, ResolverFTy->getPointerTo(GI->getAddressSpace())));
} else {
return error("Expected an alias or an ifunc");
}
@@ -2897,6 +3021,9 @@ Error BitcodeReader::parseConstants() {
case bitc::CST_CODE_NULL: // NULL
if (CurTy->isVoidTy() || CurTy->isFunctionTy() || CurTy->isLabelTy())
return error("Invalid type for a constant null value");
+ if (auto *TETy = dyn_cast<TargetExtType>(CurTy))
+ if (!TETy->hasProperty(TargetExtType::HasZeroInit))
+ return error("Invalid type for a constant null value");
V = Constant::getNullValue(CurTy);
break;
case bitc::CST_CODE_INTEGER: // INTEGER: [intval]
@@ -3111,7 +3238,7 @@ Error BitcodeReader::parseConstants() {
PointeeType = getTypeByID(Record[OpNum++]);
bool InBounds = false;
- Optional<unsigned> InRangeIndex;
+ std::optional<unsigned> InRangeIndex;
if (BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX) {
uint64_t Op = Record[OpNum++];
InBounds = Op & 1;
@@ -3443,7 +3570,7 @@ Error BitcodeReader::parseUseLists() {
break;
case bitc::USELIST_CODE_BB:
IsBB = true;
- LLVM_FALLTHROUGH;
+ [[fallthrough]];
case bitc::USELIST_CODE_DEFAULT: {
unsigned RecordLength = Record.size();
if (RecordLength < 3)
@@ -3555,11 +3682,6 @@ Error BitcodeReader::globalCleanup() {
Function *NewFn;
if (UpgradeIntrinsicFunction(&F, NewFn))
UpgradedIntrinsics[&F] = NewFn;
- else if (auto Remangled = Intrinsic::remangleIntrinsicFunction(&F))
- // Some types could be renamed during loading if several modules are
- // loaded in the same LLVMContext (LTO scenario). In this case we should
- // remangle intrinsics names as well.
- RemangledIntrinsics[&F] = *Remangled;
// Look for functions that rely on old function attribute behavior.
UpgradeFunctionAttributes(F);
}
@@ -3625,11 +3747,11 @@ Error BitcodeReader::rememberAndSkipFunctionBodies() {
}
Error BitcodeReaderBase::readBlockInfo() {
- Expected<Optional<BitstreamBlockInfo>> MaybeNewBlockInfo =
+ Expected<std::optional<BitstreamBlockInfo>> MaybeNewBlockInfo =
Stream.ReadBlockInfoBlock();
if (!MaybeNewBlockInfo)
return MaybeNewBlockInfo.takeError();
- Optional<BitstreamBlockInfo> NewBlockInfo =
+ std::optional<BitstreamBlockInfo> NewBlockInfo =
std::move(MaybeNewBlockInfo.get());
if (!NewBlockInfo)
return error("Malformed block");
@@ -3752,10 +3874,14 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) {
NewGV->setVisibility(Visibility);
NewGV->setUnnamedAddr(UnnamedAddr);
- if (Record.size() > 10)
- NewGV->setDLLStorageClass(getDecodedDLLStorageClass(Record[10]));
- else
+ if (Record.size() > 10) {
+ // A GlobalValue with local linkage cannot have a DLL storage class.
+ if (!NewGV->hasLocalLinkage()) {
+ NewGV->setDLLStorageClass(getDecodedDLLStorageClass(Record[10]));
+ }
+ } else {
upgradeDLLImportExportLinkage(NewGV, RawLinkage);
+ }
ValueList.push_back(NewGV, getVirtualTypeID(NewGV->getType(), TyID));
@@ -3796,6 +3922,14 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) {
return Error::success();
}
+void BitcodeReader::callValueTypeCallback(Value *F, unsigned TypeID) {
+ if (ValueTypeCallback) {
+ (*ValueTypeCallback)(
+ F, TypeID, [this](unsigned I) { return getTypeByID(I); },
+ [this](unsigned I, unsigned J) { return getContainedTypeID(I, J); });
+ }
+}
+
Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
// v1: [type, callingconv, isproto, linkage, paramattr, alignment, section,
// visibility, gc, unnamed_addr, prologuedata, dllstorageclass, comdat,
@@ -3840,6 +3974,7 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
uint64_t RawLinkage = Record[3];
Func->setLinkage(getDecodedLinkage(RawLinkage));
Func->setAttributes(getAttributes(Record[4]));
+ callValueTypeCallback(Func, FTyID);
// Upgrade any old-style byval or sret without a type by propagating the
// argument's pointee type. There should be no opaque pointers where the byval
@@ -3916,10 +4051,14 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
if (Record.size() > 10)
OperandInfo.Prologue = Record[10];
- if (Record.size() > 11)
- Func->setDLLStorageClass(getDecodedDLLStorageClass(Record[11]));
- else
+ if (Record.size() > 11) {
+ // A GlobalValue with local linkage cannot have a DLL storage class.
+ if (!Func->hasLocalLinkage()) {
+ Func->setDLLStorageClass(getDecodedDLLStorageClass(Record[11]));
+ }
+ } else {
upgradeDLLImportExportLinkage(Func, RawLinkage);
+ }
if (Record.size() > 12) {
if (unsigned ComdatID = Record[12]) {
@@ -4022,8 +4161,12 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord(
}
if (BitCode == bitc::MODULE_CODE_ALIAS ||
BitCode == bitc::MODULE_CODE_ALIAS_OLD) {
- if (OpNum != Record.size())
- NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++]));
+ if (OpNum != Record.size()) {
+ auto S = Record[OpNum++];
+ // A GlobalValue with local linkage cannot have a DLL storage class.
+ if (!NewGA->hasLocalLinkage())
+ NewGA->setDLLStorageClass(getDecodedDLLStorageClass(S));
+ }
else
upgradeDLLImportExportLinkage(NewGA, Linkage);
if (OpNum != Record.size())
@@ -4049,7 +4192,8 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord(
Error BitcodeReader::parseModule(uint64_t ResumeBit,
bool ShouldLazyLoadMetadata,
- DataLayoutCallbackTy DataLayoutCallback) {
+ ParserCallbacks Callbacks) {
+ this->ValueTypeCallback = std::move(Callbacks.ValueType);
if (ResumeBit) {
if (Error JumpFailed = Stream.JumpToBit(ResumeBit))
return JumpFailed;
@@ -4061,21 +4205,37 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
// Parts of bitcode parsing depend on the datalayout. Make sure we
// finalize the datalayout before we run any of that code.
bool ResolvedDataLayout = false;
- auto ResolveDataLayout = [&] {
+ // In order to support importing modules with illegal data layout strings,
+ // delay parsing the data layout string until after upgrades and overrides
+ // have been applied, allowing to fix illegal data layout strings.
+ // Initialize to the current module's layout string in case none is specified.
+ std::string TentativeDataLayoutStr = TheModule->getDataLayoutStr();
+
+ auto ResolveDataLayout = [&]() -> Error {
if (ResolvedDataLayout)
- return;
+ return Error::success();
- // datalayout and triple can't be parsed after this point.
+ // Datalayout and triple can't be parsed after this point.
ResolvedDataLayout = true;
- // Upgrade data layout string.
- std::string DL = llvm::UpgradeDataLayoutString(
- TheModule->getDataLayoutStr(), TheModule->getTargetTriple());
- TheModule->setDataLayout(DL);
+ // Auto-upgrade the layout string
+ TentativeDataLayoutStr = llvm::UpgradeDataLayoutString(
+ TentativeDataLayoutStr, TheModule->getTargetTriple());
+
+ // Apply override
+ if (Callbacks.DataLayout) {
+ if (auto LayoutOverride = (*Callbacks.DataLayout)(
+ TheModule->getTargetTriple(), TentativeDataLayoutStr))
+ TentativeDataLayoutStr = *LayoutOverride;
+ }
+
+ // Now the layout string is finalized in TentativeDataLayoutStr. Parse it.
+ Expected<DataLayout> MaybeDL = DataLayout::parse(TentativeDataLayoutStr);
+ if (!MaybeDL)
+ return MaybeDL.takeError();
- if (auto LayoutOverride =
- DataLayoutCallback(TheModule->getTargetTriple()))
- TheModule->setDataLayout(*LayoutOverride);
+ TheModule->setDataLayout(MaybeDL.get());
+ return Error::success();
};
// Read all the records for this module.
@@ -4089,7 +4249,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
case BitstreamEntry::Error:
return error("Malformed block");
case BitstreamEntry::EndBlock:
- ResolveDataLayout();
+ if (Error Err = ResolveDataLayout())
+ return Err;
return globalCleanup();
case BitstreamEntry::SubBlock:
@@ -4154,7 +4315,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
return Err;
break;
case bitc::FUNCTION_BLOCK_ID:
- ResolveDataLayout();
+ if (Error Err = ResolveDataLayout())
+ return Err;
// If this is the first function body we've seen, reverse the
// FunctionsWithBodies list.
@@ -4253,13 +4415,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
case bitc::MODULE_CODE_DATALAYOUT: { // DATALAYOUT: [strchr x N]
if (ResolvedDataLayout)
return error("datalayout too late in module");
- std::string S;
- if (convertToString(Record, 0, S))
+ if (convertToString(Record, 0, TentativeDataLayoutStr))
return error("Invalid record");
- Expected<DataLayout> MaybeDL = DataLayout::parse(S);
- if (!MaybeDL)
- return MaybeDL.takeError();
- TheModule->setDataLayout(MaybeDL.get());
break;
}
case bitc::MODULE_CODE_ASM: { // ASM: [strchr x N]
@@ -4305,7 +4462,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
return Err;
break;
case bitc::MODULE_CODE_FUNCTION:
- ResolveDataLayout();
+ if (Error Err = ResolveDataLayout())
+ return Err;
if (Error Err = parseFunctionRecord(Record))
return Err;
break;
@@ -4334,15 +4492,22 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
}
Record.clear();
}
+ this->ValueTypeCallback = std::nullopt;
+ return Error::success();
}
Error BitcodeReader::parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata,
bool IsImporting,
- DataLayoutCallbackTy DataLayoutCallback) {
+ ParserCallbacks Callbacks) {
TheModule = M;
- MDLoader = MetadataLoader(Stream, *M, ValueList, IsImporting,
- [&](unsigned ID) { return getTypeByID(ID); });
- return parseModule(0, ShouldLazyLoadMetadata, DataLayoutCallback);
+ MetadataLoaderCallbacks MDCallbacks;
+ MDCallbacks.GetTypeByID = [&](unsigned ID) { return getTypeByID(ID); };
+ MDCallbacks.GetContainedTypeID = [&](unsigned I, unsigned J) {
+ return getContainedTypeID(I, J);
+ };
+ MDCallbacks.MDType = Callbacks.MDType;
+ MDLoader = MetadataLoader(Stream, *M, ValueList, IsImporting, MDCallbacks);
+ return parseModule(0, ShouldLazyLoadMetadata, Callbacks);
}
Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) {
@@ -4737,7 +4902,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (Temp) {
InstructionList.push_back(Temp);
assert(CurBB && "No current BB?");
- CurBB->getInstList().push_back(Temp);
+ Temp->insertInto(CurBB, CurBB->end());
}
} else {
auto CastOp = (Instruction::CastOps)Opc;
@@ -5261,7 +5426,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
unsigned ActiveWords = 1;
if (ValueBitWidth > 64)
ActiveWords = Record[CurIdx++];
- Low = readWideAPInt(makeArrayRef(&Record[CurIdx], ActiveWords),
+ Low = readWideAPInt(ArrayRef(&Record[CurIdx], ActiveWords),
ValueBitWidth);
CurIdx += ActiveWords;
@@ -5269,8 +5434,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
ActiveWords = 1;
if (ValueBitWidth > 64)
ActiveWords = Record[CurIdx++];
- APInt High = readWideAPInt(
- makeArrayRef(&Record[CurIdx], ActiveWords), ValueBitWidth);
+ APInt High = readWideAPInt(ArrayRef(&Record[CurIdx], ActiveWords),
+ ValueBitWidth);
CurIdx += ActiveWords;
// FIXME: It is not clear whether values in the range should be
@@ -5985,7 +6150,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
// Before weak cmpxchgs existed, the instruction simply returned the
// value loaded from memory, so bitcode files from that era will be
// expecting the first component of a modern cmpxchg.
- CurBB->getInstList().push_back(I);
+ I->insertInto(CurBB, CurBB->end());
I = ExtractValueInst::Create(I, 0);
ResTypeID = CmpTypeID;
} else {
@@ -6309,7 +6474,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
I->deleteValue();
return error("Operand bundles found with no consumer");
}
- CurBB->getInstList().push_back(I);
+ I->insertInto(CurBB, CurBB->end());
// If this was a terminator instruction, move to the next block.
if (I->isTerminator()) {
@@ -6337,7 +6502,7 @@ OutOfRecordLoop:
// We found at least one unresolved value. Nuke them all to avoid leaks.
for (unsigned i = ModuleValueListSize, e = ValueList.size(); i != e; ++i){
if ((A = dyn_cast_or_null<Argument>(ValueList[i])) && !A->getParent()) {
- A->replaceAllUsesWith(UndefValue::get(A->getType()));
+ A->replaceAllUsesWith(PoisonValue::get(A->getType()));
delete A;
}
}
@@ -6434,12 +6599,6 @@ Error BitcodeReader::materialize(GlobalValue *GV) {
UpgradeIntrinsicCall(CI, I.second);
}
- // Update calls to the remangled intrinsics
- for (auto &I : RemangledIntrinsics)
- for (User *U : llvm::make_early_inc_range(I.first->materialized_users()))
- // Don't expect any other users than call sites
- cast<CallBase>(U)->setCalledFunction(I.second);
-
// Finish fn->subprogram upgrade for materialized functions.
if (DISubprogram *SP = MDLoader->lookupSubprogramForFunction(F))
F->setSubprogram(SP);
@@ -6544,12 +6703,6 @@ Error BitcodeReader::materializeModule() {
I.first->eraseFromParent();
}
UpgradedIntrinsics.clear();
- // Do the same for remangled intrinsics
- for (auto &I : RemangledIntrinsics) {
- I.first->replaceAllUsesWith(I.second);
- I.first->eraseFromParent();
- }
- RemangledIntrinsics.clear();
UpgradeDebugInfo(*TheModule);
@@ -6566,9 +6719,10 @@ std::vector<StructType *> BitcodeReader::getIdentifiedStructTypes() const {
ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader(
BitstreamCursor Cursor, StringRef Strtab, ModuleSummaryIndex &TheIndex,
- StringRef ModulePath, unsigned ModuleId)
+ StringRef ModulePath, unsigned ModuleId,
+ std::function<bool(GlobalValue::GUID)> IsPrevailing)
: BitcodeReaderBase(std::move(Cursor), Strtab), TheIndex(TheIndex),
- ModulePath(ModulePath), ModuleId(ModuleId) {}
+ ModulePath(ModulePath), ModuleId(ModuleId), IsPrevailing(IsPrevailing) {}
void ModuleSummaryIndexBitcodeReader::addThisModule() {
TheIndex.addModule(ModulePath, ModuleId);
@@ -6579,10 +6733,15 @@ ModuleSummaryIndexBitcodeReader::getThisModule() {
return TheIndex.getModule(ModulePath);
}
-std::pair<ValueInfo, GlobalValue::GUID>
+template <bool AllowNullValueInfo>
+std::tuple<ValueInfo, GlobalValue::GUID, GlobalValue::GUID>
ModuleSummaryIndexBitcodeReader::getValueInfoFromValueId(unsigned ValueId) {
auto VGI = ValueIdToValueInfoMap[ValueId];
- assert(VGI.first);
+ // We can have a null value info for memprof callsite info records in
+ // distributed ThinLTO index files when the callee function summary is not
+ // included in the index. The bitcode writer records 0 in that case,
+ // and the caller of this helper will set AllowNullValueInfo to true.
+ assert(AllowNullValueInfo || std::get<0>(VGI));
return VGI;
}
@@ -6602,11 +6761,10 @@ void ModuleSummaryIndexBitcodeReader::setValueGUID(
// UseStrtab is false for legacy summary formats and value names are
// created on stack. In that case we save the name in a string saver in
// the index so that the value name can be recorded.
- ValueIdToValueInfoMap[ValueID] = std::make_pair(
+ ValueIdToValueInfoMap[ValueID] = std::make_tuple(
TheIndex.getOrInsertValueInfo(
- ValueGUID,
- UseStrtab ? ValueName : TheIndex.saveString(ValueName)),
- OriginalNameID);
+ ValueGUID, UseStrtab ? ValueName : TheIndex.saveString(ValueName)),
+ OriginalNameID, ValueGUID);
}
// Specialized value symbol table parser used when reading module index
@@ -6694,8 +6852,8 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable(
GlobalValue::GUID RefGUID = Record[1];
// The "original name", which is the second value of the pair will be
// overriden later by a FS_COMBINED_ORIGINAL_NAME in the combined index.
- ValueIdToValueInfoMap[ValueID] =
- std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID);
+ ValueIdToValueInfoMap[ValueID] = std::make_tuple(
+ TheIndex.getOrInsertValueInfo(RefGUID), RefGUID, RefGUID);
break;
}
}
@@ -6849,7 +7007,7 @@ ModuleSummaryIndexBitcodeReader::makeRefList(ArrayRef<uint64_t> Record) {
std::vector<ValueInfo> Ret;
Ret.reserve(Record.size());
for (uint64_t RefValueId : Record)
- Ret.push_back(getValueInfoFromValueId(RefValueId).first);
+ Ret.push_back(std::get<0>(getValueInfoFromValueId(RefValueId)));
return Ret;
}
@@ -6862,7 +7020,7 @@ ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef<uint64_t> Record,
for (unsigned I = 0, E = Record.size(); I != E; ++I) {
CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown;
uint64_t RelBF = 0;
- ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
+ ValueInfo Callee = std::get<0>(getValueInfoFromValueId(Record[I]));
if (IsOldProfileFormat) {
I += 1; // Skip old callsitecount field
if (HasProfile)
@@ -6953,7 +7111,7 @@ ModuleSummaryIndexBitcodeReader::parseParamAccesses(ArrayRef<uint64_t> Record) {
for (auto &Call : ParamAccess.Calls) {
Call.ParamNo = Record.front();
Record = Record.drop_front();
- Call.Callee = getValueInfoFromValueId(Record.front()).first;
+ Call.Callee = std::get<0>(getValueInfoFromValueId(Record.front()));
Record = Record.drop_front();
Call.Offsets = ReadRange();
}
@@ -6965,7 +7123,7 @@ void ModuleSummaryIndexBitcodeReader::parseTypeIdCompatibleVtableInfo(
ArrayRef<uint64_t> Record, size_t &Slot,
TypeIdCompatibleVtableInfo &TypeId) {
uint64_t Offset = Record[Slot++];
- ValueInfo Callee = getValueInfoFromValueId(Record[Slot++]).first;
+ ValueInfo Callee = std::get<0>(getValueInfoFromValueId(Record[Slot++]));
TypeId.push_back({Offset, Callee});
}
@@ -7040,6 +7198,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
PendingTypeCheckedLoadConstVCalls;
std::vector<FunctionSummary::ParamAccess> PendingParamAccesses;
+ std::vector<CallsiteInfo> PendingCallsites;
+ std::vector<AllocInfo> PendingAllocs;
+
while (true) {
Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();
if (!MaybeEntry)
@@ -7078,8 +7239,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
case bitc::FS_VALUE_GUID: { // [valueid, refguid]
uint64_t ValueID = Record[0];
GlobalValue::GUID RefGUID = Record[1];
- ValueIdToValueInfoMap[ValueID] =
- std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID);
+ ValueIdToValueInfoMap[ValueID] = std::make_tuple(
+ TheIndex.getOrInsertValueInfo(RefGUID), RefGUID, RefGUID);
break;
}
// FS_PERMODULE: [valueid, flags, instcount, fflags, numrefs,
@@ -7131,6 +7292,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
IsOldProfileFormat, HasProfile, HasRelBF);
setSpecialRefs(Refs, NumRORefs, NumWORefs);
+ auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID);
+ // In order to save memory, only record the memprof summaries if this is
+ // the prevailing copy of a symbol. The linker doesn't resolve local
+ // linkage values so don't check whether those are prevailing.
+ auto LT = (GlobalValue::LinkageTypes)Flags.Linkage;
+ if (IsPrevailing &&
+ !GlobalValue::isLocalLinkage(LT) &&
+ !IsPrevailing(std::get<2>(VIAndOriginalGUID))) {
+ PendingCallsites.clear();
+ PendingAllocs.clear();
+ }
auto FS = std::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), /*EntryCount=*/0,
std::move(Refs), std::move(Calls), std::move(PendingTypeTests),
@@ -7138,11 +7310,12 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::move(PendingTypeCheckedLoadVCalls),
std::move(PendingTypeTestAssumeConstVCalls),
std::move(PendingTypeCheckedLoadConstVCalls),
- std::move(PendingParamAccesses));
- auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID);
+ std::move(PendingParamAccesses), std::move(PendingCallsites),
+ std::move(PendingAllocs));
FS->setModulePath(getThisModule()->first());
- FS->setOriginalName(VIAndOriginalGUID.second);
- TheIndex.addGlobalValueSummary(VIAndOriginalGUID.first, std::move(FS));
+ FS->setOriginalName(std::get<1>(VIAndOriginalGUID));
+ TheIndex.addGlobalValueSummary(std::get<0>(VIAndOriginalGUID),
+ std::move(FS));
break;
}
// FS_ALIAS: [valueid, flags, valueid]
@@ -7161,15 +7334,15 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
// ownership.
AS->setModulePath(getThisModule()->first());
- auto AliaseeVI = getValueInfoFromValueId(AliaseeID).first;
+ auto AliaseeVI = std::get<0>(getValueInfoFromValueId(AliaseeID));
auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeVI, ModulePath);
if (!AliaseeInModule)
return error("Alias expects aliasee summary to be parsed");
AS->setAliasee(AliaseeVI, AliaseeInModule);
auto GUID = getValueInfoFromValueId(ValueID);
- AS->setOriginalName(GUID.second);
- TheIndex.addGlobalValueSummary(GUID.first, std::move(AS));
+ AS->setOriginalName(std::get<1>(GUID));
+ TheIndex.addGlobalValueSummary(std::get<0>(GUID), std::move(AS));
break;
}
// FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, n x valueid]
@@ -7192,8 +7365,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));
FS->setModulePath(getThisModule()->first());
auto GUID = getValueInfoFromValueId(ValueID);
- FS->setOriginalName(GUID.second);
- TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
+ FS->setOriginalName(std::get<1>(GUID));
+ TheIndex.addGlobalValueSummary(std::get<0>(GUID), std::move(FS));
break;
}
// FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags,
@@ -7211,7 +7384,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs));
VTableFuncList VTableFuncs;
for (unsigned I = VTableListStartIndex, E = Record.size(); I != E; ++I) {
- ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
+ ValueInfo Callee = std::get<0>(getValueInfoFromValueId(Record[I]));
uint64_t Offset = Record[++I];
VTableFuncs.push_back({Callee, Offset});
}
@@ -7220,8 +7393,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
VS->setModulePath(getThisModule()->first());
VS->setVTableFuncs(VTableFuncs);
auto GUID = getValueInfoFromValueId(ValueID);
- VS->setOriginalName(GUID.second);
- TheIndex.addGlobalValueSummary(GUID.first, std::move(VS));
+ VS->setOriginalName(std::get<1>(GUID));
+ TheIndex.addGlobalValueSummary(std::get<0>(GUID), std::move(VS));
break;
}
// FS_COMBINED: [valueid, modid, flags, instcount, fflags, numrefs,
@@ -7272,7 +7445,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::vector<FunctionSummary::EdgeTy> Edges = makeCallList(
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
IsOldProfileFormat, HasProfile, false);
- ValueInfo VI = getValueInfoFromValueId(ValueID).first;
+ ValueInfo VI = std::get<0>(getValueInfoFromValueId(ValueID));
setSpecialRefs(Refs, NumRORefs, NumWORefs);
auto FS = std::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), EntryCount,
@@ -7281,7 +7454,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::move(PendingTypeCheckedLoadVCalls),
std::move(PendingTypeTestAssumeConstVCalls),
std::move(PendingTypeCheckedLoadConstVCalls),
- std::move(PendingParamAccesses));
+ std::move(PendingParamAccesses), std::move(PendingCallsites),
+ std::move(PendingAllocs));
LastSeenSummary = FS.get();
LastSeenGUID = VI.getGUID();
FS->setModulePath(ModuleIdMap[ModuleId]);
@@ -7301,11 +7475,11 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
LastSeenSummary = AS.get();
AS->setModulePath(ModuleIdMap[ModuleId]);
- auto AliaseeVI = getValueInfoFromValueId(AliaseeValueId).first;
+ auto AliaseeVI = std::get<0>(getValueInfoFromValueId(AliaseeValueId));
auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeVI, AS->modulePath());
AS->setAliasee(AliaseeVI, AliaseeInModule);
- ValueInfo VI = getValueInfoFromValueId(ValueID).first;
+ ValueInfo VI = std::get<0>(getValueInfoFromValueId(ValueID));
LastSeenGUID = VI.getGUID();
TheIndex.addGlobalValueSummary(VI, std::move(AS));
break;
@@ -7331,7 +7505,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));
LastSeenSummary = FS.get();
FS->setModulePath(ModuleIdMap[ModuleId]);
- ValueInfo VI = getValueInfoFromValueId(ValueID).first;
+ ValueInfo VI = std::get<0>(getValueInfoFromValueId(ValueID));
LastSeenGUID = VI.getGUID();
TheIndex.addGlobalValueSummary(VI, std::move(FS));
break;
@@ -7407,6 +7581,95 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
PendingParamAccesses = parseParamAccesses(Record);
break;
}
+
+ case bitc::FS_STACK_IDS: { // [n x stackid]
+ // Save stack ids in the reader to consult when adding stack ids from the
+ // lists in the stack node and alloc node entries.
+ StackIds = ArrayRef<uint64_t>(Record);
+ break;
+ }
+
+ case bitc::FS_PERMODULE_CALLSITE_INFO: {
+ unsigned ValueID = Record[0];
+ SmallVector<unsigned> StackIdList;
+ for (auto R = Record.begin() + 1; R != Record.end(); R++) {
+ assert(*R < StackIds.size());
+ StackIdList.push_back(TheIndex.addOrGetStackIdIndex(StackIds[*R]));
+ }
+ ValueInfo VI = std::get<0>(getValueInfoFromValueId(ValueID));
+ PendingCallsites.push_back(CallsiteInfo({VI, std::move(StackIdList)}));
+ break;
+ }
+
+ case bitc::FS_COMBINED_CALLSITE_INFO: {
+ auto RecordIter = Record.begin();
+ unsigned ValueID = *RecordIter++;
+ unsigned NumStackIds = *RecordIter++;
+ unsigned NumVersions = *RecordIter++;
+ assert(Record.size() == 3 + NumStackIds + NumVersions);
+ SmallVector<unsigned> StackIdList;
+ for (unsigned J = 0; J < NumStackIds; J++) {
+ assert(*RecordIter < StackIds.size());
+ StackIdList.push_back(
+ TheIndex.addOrGetStackIdIndex(StackIds[*RecordIter++]));
+ }
+ SmallVector<unsigned> Versions;
+ for (unsigned J = 0; J < NumVersions; J++)
+ Versions.push_back(*RecordIter++);
+ ValueInfo VI = std::get<0>(
+ getValueInfoFromValueId</*AllowNullValueInfo*/ true>(ValueID));
+ PendingCallsites.push_back(
+ CallsiteInfo({VI, std::move(Versions), std::move(StackIdList)}));
+ break;
+ }
+
+ case bitc::FS_PERMODULE_ALLOC_INFO: {
+ unsigned I = 0;
+ std::vector<MIBInfo> MIBs;
+ while (I < Record.size()) {
+ assert(Record.size() - I >= 2);
+ AllocationType AllocType = (AllocationType)Record[I++];
+ unsigned NumStackEntries = Record[I++];
+ assert(Record.size() - I >= NumStackEntries);
+ SmallVector<unsigned> StackIdList;
+ for (unsigned J = 0; J < NumStackEntries; J++) {
+ assert(Record[I] < StackIds.size());
+ StackIdList.push_back(
+ TheIndex.addOrGetStackIdIndex(StackIds[Record[I++]]));
+ }
+ MIBs.push_back(MIBInfo(AllocType, std::move(StackIdList)));
+ }
+ PendingAllocs.push_back(AllocInfo(std::move(MIBs)));
+ break;
+ }
+
+ case bitc::FS_COMBINED_ALLOC_INFO: {
+ unsigned I = 0;
+ std::vector<MIBInfo> MIBs;
+ unsigned NumMIBs = Record[I++];
+ unsigned NumVersions = Record[I++];
+ unsigned MIBsRead = 0;
+ while (MIBsRead++ < NumMIBs) {
+ assert(Record.size() - I >= 2);
+ AllocationType AllocType = (AllocationType)Record[I++];
+ unsigned NumStackEntries = Record[I++];
+ assert(Record.size() - I >= NumStackEntries);
+ SmallVector<unsigned> StackIdList;
+ for (unsigned J = 0; J < NumStackEntries; J++) {
+ assert(Record[I] < StackIds.size());
+ StackIdList.push_back(
+ TheIndex.addOrGetStackIdIndex(StackIds[Record[I++]]));
+ }
+ MIBs.push_back(MIBInfo(AllocType, std::move(StackIdList)));
+ }
+ assert(Record.size() - I >= NumVersions);
+ SmallVector<uint8_t> Versions;
+ for (unsigned J = 0; J < NumVersions; J++)
+ Versions.push_back(Record[I++]);
+ PendingAllocs.push_back(
+ AllocInfo(std::move(Versions), std::move(MIBs)));
+ break;
+ }
}
}
llvm_unreachable("Exit infinite loop");
@@ -7677,7 +7940,7 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) {
Expected<std::unique_ptr<Module>>
BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll,
bool ShouldLazyLoadMetadata, bool IsImporting,
- DataLayoutCallbackTy DataLayoutCallback) {
+ ParserCallbacks Callbacks) {
BitstreamCursor Stream(Buffer);
std::string ProducerIdentification;
@@ -7700,7 +7963,7 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll,
// Delay parsing Metadata if ShouldLazyLoadMetadata is true.
if (Error Err = R->parseBitcodeInto(M.get(), ShouldLazyLoadMetadata,
- IsImporting, DataLayoutCallback))
+ IsImporting, Callbacks))
return std::move(Err);
if (MaterializeAll) {
@@ -7717,23 +7980,24 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll,
Expected<std::unique_ptr<Module>>
BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata,
- bool IsImporting) {
+ bool IsImporting, ParserCallbacks Callbacks) {
return getModuleImpl(Context, false, ShouldLazyLoadMetadata, IsImporting,
- [](StringRef) { return None; });
+ Callbacks);
}
// Parse the specified bitcode buffer and merge the index into CombinedIndex.
// We don't use ModuleIdentifier here because the client may need to control the
// module path used in the combined summary (e.g. when reading summaries for
// regular LTO modules).
-Error BitcodeModule::readSummary(ModuleSummaryIndex &CombinedIndex,
- StringRef ModulePath, uint64_t ModuleId) {
+Error BitcodeModule::readSummary(
+ ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, uint64_t ModuleId,
+ std::function<bool(GlobalValue::GUID)> IsPrevailing) {
BitstreamCursor Stream(Buffer);
if (Error JumpFailed = Stream.JumpToBit(ModuleBit))
return JumpFailed;
ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, CombinedIndex,
- ModulePath, ModuleId);
+ ModulePath, ModuleId, IsPrevailing);
return R.parseModule();
}
@@ -7864,40 +8128,41 @@ static Expected<BitcodeModule> getSingleModule(MemoryBufferRef Buffer) {
Expected<std::unique_ptr<Module>>
llvm::getLazyBitcodeModule(MemoryBufferRef Buffer, LLVMContext &Context,
- bool ShouldLazyLoadMetadata, bool IsImporting) {
+ bool ShouldLazyLoadMetadata, bool IsImporting,
+ ParserCallbacks Callbacks) {
Expected<BitcodeModule> BM = getSingleModule(Buffer);
if (!BM)
return BM.takeError();
- return BM->getLazyModule(Context, ShouldLazyLoadMetadata, IsImporting);
+ return BM->getLazyModule(Context, ShouldLazyLoadMetadata, IsImporting,
+ Callbacks);
}
Expected<std::unique_ptr<Module>> llvm::getOwningLazyBitcodeModule(
std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext &Context,
- bool ShouldLazyLoadMetadata, bool IsImporting) {
+ bool ShouldLazyLoadMetadata, bool IsImporting, ParserCallbacks Callbacks) {
auto MOrErr = getLazyBitcodeModule(*Buffer, Context, ShouldLazyLoadMetadata,
- IsImporting);
+ IsImporting, Callbacks);
if (MOrErr)
(*MOrErr)->setOwnedMemoryBuffer(std::move(Buffer));
return MOrErr;
}
Expected<std::unique_ptr<Module>>
-BitcodeModule::parseModule(LLVMContext &Context,
- DataLayoutCallbackTy DataLayoutCallback) {
- return getModuleImpl(Context, true, false, false, DataLayoutCallback);
+BitcodeModule::parseModule(LLVMContext &Context, ParserCallbacks Callbacks) {
+ return getModuleImpl(Context, true, false, false, Callbacks);
// TODO: Restore the use-lists to the in-memory state when the bitcode was
// written. We must defer until the Module has been fully materialized.
}
Expected<std::unique_ptr<Module>>
llvm::parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context,
- DataLayoutCallbackTy DataLayoutCallback) {
+ ParserCallbacks Callbacks) {
Expected<BitcodeModule> BM = getSingleModule(Buffer);
if (!BM)
return BM.takeError();
- return BM->parseModule(Context, DataLayoutCallback);
+ return BM->parseModule(Context, Callbacks);
}
Expected<std::string> llvm::getBitcodeTargetTriple(MemoryBufferRef Buffer) {