diff options
Diffstat (limited to 'llvm/lib/Bitcode/Reader')
-rw-r--r-- | llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp | 5 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 291 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/MetadataLoader.cpp | 192 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/ValueList.cpp | 3 |
4 files changed, 365 insertions, 126 deletions
diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp index 2ce064c7685a..e91af121ea08 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -135,6 +135,7 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(MODULE_CODE, FUNCTION) STRINGIFY_CODE(MODULE_CODE, ALIAS) STRINGIFY_CODE(MODULE_CODE, GCNAME) + STRINGIFY_CODE(MODULE_CODE, COMDAT) STRINGIFY_CODE(MODULE_CODE, VSTOFFSET) STRINGIFY_CODE(MODULE_CODE, METADATA_VALUES_UNUSED) STRINGIFY_CODE(MODULE_CODE, SOURCE_FILENAME) @@ -176,16 +177,20 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(TYPE_CODE, OPAQUE) STRINGIFY_CODE(TYPE_CODE, INTEGER) STRINGIFY_CODE(TYPE_CODE, POINTER) + STRINGIFY_CODE(TYPE_CODE, HALF) STRINGIFY_CODE(TYPE_CODE, ARRAY) STRINGIFY_CODE(TYPE_CODE, VECTOR) STRINGIFY_CODE(TYPE_CODE, X86_FP80) STRINGIFY_CODE(TYPE_CODE, FP128) STRINGIFY_CODE(TYPE_CODE, PPC_FP128) STRINGIFY_CODE(TYPE_CODE, METADATA) + STRINGIFY_CODE(TYPE_CODE, X86_MMX) STRINGIFY_CODE(TYPE_CODE, STRUCT_ANON) STRINGIFY_CODE(TYPE_CODE, STRUCT_NAME) STRINGIFY_CODE(TYPE_CODE, STRUCT_NAMED) STRINGIFY_CODE(TYPE_CODE, FUNCTION) + STRINGIFY_CODE(TYPE_CODE, TOKEN) + STRINGIFY_CODE(TYPE_CODE, BFLOAT) } case bitc::CONSTANTS_BLOCK_ID: diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 659e26c2bd25..f2800201e871 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -20,8 +20,9 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" -#include "llvm/Bitstream/BitstreamReader.h" +#include "llvm/Bitcode/BitcodeCommon.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" @@ -578,9 +579,7 @@ public: /// \returns true if an error occurred. Error parseBitcodeInto( Module *M, bool ShouldLazyLoadMetadata = false, bool IsImporting = false, - DataLayoutCallbackTy DataLayoutCallback = [](std::string) { - return None; - }); + DataLayoutCallbackTy DataLayoutCallback = [](StringRef) { return None; }); static uint64_t decodeSignRotatedValue(uint64_t V); @@ -650,7 +649,7 @@ private: /// Read a value/type pair out of the specified record from slot 'Slot'. /// Increment Slot past the number of slots used in the record. Return true on /// failure. - bool getValueTypePair(SmallVectorImpl<uint64_t> &Record, unsigned &Slot, + bool getValueTypePair(const SmallVectorImpl<uint64_t> &Record, unsigned &Slot, unsigned InstNum, Value *&ResVal, Type **FullTy = nullptr) { if (Slot == Record.size()) return true; @@ -677,7 +676,7 @@ private: /// Read a value out of the specified record from slot 'Slot'. Increment Slot /// past the number of slots used by the value in the record. Return true if /// there is an error. - bool popValue(SmallVectorImpl<uint64_t> &Record, unsigned &Slot, + bool popValue(const SmallVectorImpl<uint64_t> &Record, unsigned &Slot, unsigned InstNum, Type *Ty, Value *&ResVal) { if (getValue(Record, Slot, InstNum, Ty, ResVal)) return true; @@ -687,7 +686,7 @@ private: } /// Like popValue, but does not increment the Slot number. - bool getValue(SmallVectorImpl<uint64_t> &Record, unsigned Slot, + bool getValue(const SmallVectorImpl<uint64_t> &Record, unsigned Slot, unsigned InstNum, Type *Ty, Value *&ResVal) { ResVal = getValue(Record, Slot, InstNum, Ty); return ResVal == nullptr; @@ -695,7 +694,7 @@ private: /// Version of getValue that returns ResVal directly, or 0 if there is an /// error. - Value *getValue(SmallVectorImpl<uint64_t> &Record, unsigned Slot, + Value *getValue(const SmallVectorImpl<uint64_t> &Record, unsigned Slot, unsigned InstNum, Type *Ty) { if (Slot == Record.size()) return nullptr; unsigned ValNo = (unsigned)Record[Slot]; @@ -706,7 +705,7 @@ private: } /// Like getValue, but decodes signed VBRs. - Value *getValueSigned(SmallVectorImpl<uint64_t> &Record, unsigned Slot, + Value *getValueSigned(const SmallVectorImpl<uint64_t> &Record, unsigned Slot, unsigned InstNum, Type *Ty) { if (Slot == Record.size()) return nullptr; unsigned ValNo = (unsigned)decodeSignRotatedValue(Record[Slot]); @@ -716,9 +715,9 @@ private: return getFnValueByID(ValNo, Ty); } - /// Upgrades old-style typeless byval attributes by adding the corresponding - /// argument's pointee type. - void propagateByValTypes(CallBase *CB, ArrayRef<Type *> ArgsFullTys); + /// Upgrades old-style typeless byval or sret attributes by adding the + /// corresponding argument's pointee type. + void propagateByValSRetTypes(CallBase *CB, ArrayRef<Type *> ArgsFullTys); /// Converts alignment exponent (i.e. power of two (or zero)) to the /// corresponding alignment to use. If alignment is too large, returns @@ -834,6 +833,8 @@ private: void parseTypeIdCompatibleVtableSummaryRecord(ArrayRef<uint64_t> Record); void parseTypeIdCompatibleVtableInfo(ArrayRef<uint64_t> Record, size_t &Slot, TypeIdCompatibleVtableInfo &TypeId); + std::vector<FunctionSummary::ParamAccess> + parseParamAccesses(ArrayRef<uint64_t> Record); std::pair<ValueInfo, GlobalValue::GUID> getValueInfoFromValueId(unsigned ValueId); @@ -1432,6 +1433,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoAlias; case bitc::ATTR_KIND_NO_BUILTIN: return Attribute::NoBuiltin; + case bitc::ATTR_KIND_NO_CALLBACK: + return Attribute::NoCallback; case bitc::ATTR_KIND_NO_CAPTURE: return Attribute::NoCapture; case bitc::ATTR_KIND_NO_DUPLICATE: @@ -1532,6 +1535,12 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::Preallocated; case bitc::ATTR_KIND_NOUNDEF: return Attribute::NoUndef; + case bitc::ATTR_KIND_BYREF: + return Attribute::ByRef; + case bitc::ATTR_KIND_MUSTPROGRESS: + return Attribute::MustProgress; + case bitc::ATTR_KIND_HOT: + return Attribute::Hot; } } @@ -1606,6 +1615,8 @@ Error BitcodeReader::parseAttributeGroupBlock() { // this AttributeList with a function. if (Kind == Attribute::ByVal) B.addByValAttr(nullptr); + else if (Kind == Attribute::StructRet) + B.addStructRetAttr(nullptr); B.addAttribute(Kind); } else if (Record[i] == 1) { // Integer attribute @@ -1649,6 +1660,10 @@ Error BitcodeReader::parseAttributeGroupBlock() { return Err; if (Kind == Attribute::ByVal) { B.addByValAttr(HasType ? getTypeByID(Record[++i]) : nullptr); + } else if (Kind == Attribute::StructRet) { + B.addStructRetAttr(HasType ? getTypeByID(Record[++i]) : nullptr); + } else if (Kind == Attribute::ByRef) { + B.addByRefAttr(getTypeByID(Record[++i])); } else if (Kind == Attribute::Preallocated) { B.addPreallocatedAttr(getTypeByID(Record[++i])); } @@ -1711,7 +1726,7 @@ Error BitcodeReader::parseTypeTableBody() { case bitc::TYPE_CODE_NUMENTRY: // TYPE_CODE_NUMENTRY: [numentries] // TYPE_CODE_NUMENTRY contains a count of the number of types in the // type list. This allows us to reserve space. - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); TypeList.resize(Record[0]); continue; @@ -1748,11 +1763,14 @@ Error BitcodeReader::parseTypeTableBody() { case bitc::TYPE_CODE_X86_MMX: // X86_MMX ResultTy = Type::getX86_MMXTy(Context); break; + case bitc::TYPE_CODE_X86_AMX: // X86_AMX + ResultTy = Type::getX86_AMXTy(Context); + break; case bitc::TYPE_CODE_TOKEN: // TOKEN ResultTy = Type::getTokenTy(Context); break; case bitc::TYPE_CODE_INTEGER: { // INTEGER: [width] - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); uint64_t NumBits = Record[0]; @@ -1764,7 +1782,7 @@ Error BitcodeReader::parseTypeTableBody() { } case bitc::TYPE_CODE_POINTER: { // POINTER: [pointee type] or // [pointee type, address space] - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); unsigned AddressSpace = 0; if (Record.size() == 2) @@ -1819,7 +1837,7 @@ Error BitcodeReader::parseTypeTableBody() { break; } case bitc::TYPE_CODE_STRUCT_ANON: { // STRUCT: [ispacked, eltty x N] - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); SmallVector<Type*, 8> EltTys; for (unsigned i = 1, e = Record.size(); i != e; ++i) { @@ -1839,7 +1857,7 @@ Error BitcodeReader::parseTypeTableBody() { continue; case bitc::TYPE_CODE_STRUCT_NAMED: { // STRUCT: [ispacked, eltty x N] - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); if (NumRecords >= TypeList.size()) @@ -2399,6 +2417,9 @@ Error BitcodeReader::parseConstants() { case bitc::CST_CODE_UNDEF: // UNDEF V = UndefValue::get(CurTy); break; + case bitc::CST_CODE_POISON: // POISON + V = PoisonValue::get(CurTy); + break; case bitc::CST_CODE_SETTYPE: // SETTYPE: [typeid] if (Record.empty()) return error("Invalid record"); @@ -2912,8 +2933,7 @@ Error BitcodeReader::parseUseLists() { if (RecordLength < 3) // Records should have at least an ID and two indexes. return error("Invalid record"); - unsigned ID = Record.back(); - Record.pop_back(); + unsigned ID = Record.pop_back_val(); Value *V; if (IsBB) { @@ -2965,12 +2985,15 @@ Error BitcodeReader::materializeMetadata() { } // Upgrade "Linker Options" module flag to "llvm.linker.options" module-level - // metadata. - if (Metadata *Val = TheModule->getModuleFlag("Linker Options")) { - NamedMDNode *LinkerOpts = - TheModule->getOrInsertNamedMetadata("llvm.linker.options"); - for (const MDOperand &MDOptions : cast<MDNode>(Val)->operands()) - LinkerOpts->addOperand(cast<MDNode>(MDOptions)); + // metadata. Only upgrade if the new option doesn't exist to avoid upgrade + // multiple times. + if (!TheModule->getNamedMetadata("llvm.linker.options")) { + if (Metadata *Val = TheModule->getModuleFlag("Linker Options")) { + NamedMDNode *LinkerOpts = + TheModule->getOrInsertNamedMetadata("llvm.linker.options"); + for (const MDOperand &MDOptions : cast<MDNode>(Val)->operands()) + LinkerOpts->addOperand(cast<MDNode>(MDOptions)); + } } DeferredMetadataInfo.clear(); @@ -3278,17 +3301,24 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) { Func->setLinkage(getDecodedLinkage(RawLinkage)); Func->setAttributes(getAttributes(Record[4])); - // Upgrade any old-style byval without a type by propagating the argument's - // pointee type. There should be no opaque pointers where the byval type is - // implicit. + // Upgrade any old-style byval or sret without a type by propagating the + // argument's pointee type. There should be no opaque pointers where the byval + // type is implicit. for (unsigned i = 0; i != Func->arg_size(); ++i) { - if (!Func->hasParamAttribute(i, Attribute::ByVal)) - continue; + for (Attribute::AttrKind Kind : {Attribute::ByVal, Attribute::StructRet}) { + if (!Func->hasParamAttribute(i, Kind)) + continue; - Type *PTy = cast<FunctionType>(FullFTy)->getParamType(i); - Func->removeParamAttr(i, Attribute::ByVal); - Func->addParamAttr(i, Attribute::getWithByValType( - Context, getPointerElementFlatType(PTy))); + Func->removeParamAttr(i, Kind); + + Type *PTy = cast<FunctionType>(FullFTy)->getParamType(i); + Type *PtrEltTy = getPointerElementFlatType(PTy); + Attribute NewAttr = + Kind == Attribute::ByVal + ? Attribute::getWithByValType(Context, PtrEltTy) + : Attribute::getWithStructRetType(Context, PtrEltTy); + Func->addParamAttr(i, NewAttr); + } } MaybeAlign Alignment; @@ -3708,7 +3738,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, break; /// MODULE_CODE_VSTOFFSET: [offset] case bitc::MODULE_CODE_VSTOFFSET: - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); // Note that we subtract 1 here because the offset is relative to one word // before the start of the identification or module block, which was @@ -3749,16 +3779,22 @@ Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) { return Error::success(); } -void BitcodeReader::propagateByValTypes(CallBase *CB, - ArrayRef<Type *> ArgsFullTys) { +void BitcodeReader::propagateByValSRetTypes(CallBase *CB, + ArrayRef<Type *> ArgsFullTys) { for (unsigned i = 0; i != CB->arg_size(); ++i) { - if (!CB->paramHasAttr(i, Attribute::ByVal)) - continue; + for (Attribute::AttrKind Kind : {Attribute::ByVal, Attribute::StructRet}) { + if (!CB->paramHasAttr(i, Kind)) + continue; + + CB->removeParamAttr(i, Kind); - CB->removeParamAttr(i, Attribute::ByVal); - CB->addParamAttr( - i, Attribute::getWithByValType( - Context, getPointerElementFlatType(ArgsFullTys[i]))); + Type *PtrEltTy = getPointerElementFlatType(ArgsFullTys[i]); + Attribute NewAttr = + Kind == Attribute::ByVal + ? Attribute::getWithByValType(Context, PtrEltTy) + : Attribute::getWithStructRetType(Context, PtrEltTy); + CB->addParamAttr(i, NewAttr); + } } } @@ -3862,7 +3898,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { default: // Default behavior: reject return error("Invalid value"); case bitc::FUNC_CODE_DECLAREBLOCKS: { // DECLAREBLOCKS: [nblocks] - if (Record.size() < 1 || Record[0] == 0) + if (Record.empty() || Record[0] == 0) return error("Invalid record"); // Create all the basic blocks for the function. FunctionBBs.resize(Record[0]); @@ -3929,7 +3965,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (!IA) return error("Invalid record"); } - LastLoc = DebugLoc::get(Line, Col, Scope, IA, isImplicitCode); + LastLoc = DILocation::get(Scope->getContext(), Line, Col, Scope, IA, + isImplicitCode); I->setDebugLoc(LastLoc); I = nullptr; continue; @@ -4610,7 +4647,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { cast<InvokeInst>(I)->setCallingConv( static_cast<CallingConv::ID>(CallingConv::MaxID & CCInfo)); cast<InvokeInst>(I)->setAttributes(PAL); - propagateByValTypes(cast<CallBase>(I), ArgsFullTys); + propagateByValSRetTypes(cast<CallBase>(I), ArgsFullTys); break; } @@ -4704,7 +4741,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { InstructionList.push_back(I); break; case bitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...] - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); // The first record specifies the type. FullTy = getFullyStructuredTypeByID(Record[0]); @@ -4806,17 +4843,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { case bitc::FUNC_CODE_INST_ALLOCA: { // ALLOCA: [instty, opty, op, align] if (Record.size() != 4) return error("Invalid record"); - uint64_t AlignRecord = Record[3]; - const uint64_t InAllocaMask = uint64_t(1) << 5; - const uint64_t ExplicitTypeMask = uint64_t(1) << 6; - const uint64_t SwiftErrorMask = uint64_t(1) << 7; - const uint64_t FlagMask = InAllocaMask | ExplicitTypeMask | - SwiftErrorMask; - bool InAlloca = AlignRecord & InAllocaMask; - bool SwiftError = AlignRecord & SwiftErrorMask; + using APV = AllocaPackedValues; + const uint64_t Rec = Record[3]; + const bool InAlloca = Bitfield::get<APV::UsedWithInAlloca>(Rec); + const bool SwiftError = Bitfield::get<APV::SwiftError>(Rec); FullTy = getFullyStructuredTypeByID(Record[0]); Type *Ty = flattenPointerTypes(FullTy); - if ((AlignRecord & ExplicitTypeMask) == 0) { + if (!Bitfield::get<APV::ExplicitType>(Rec)) { auto *PTy = dyn_cast_or_null<PointerType>(Ty); if (!PTy) return error("Old-style alloca with a non-pointer type"); @@ -4825,7 +4858,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Type *OpTy = getTypeByID(Record[1]); Value *Size = getFnValueByID(Record[2], OpTy); MaybeAlign Align; - if (Error Err = parseAlignmentValue(AlignRecord & ~FlagMask, Align)) { + if (Error Err = + parseAlignmentValue(Bitfield::get<APV::Align>(Rec), Align)) { return Err; } if (!Ty || !Size) @@ -4982,54 +5016,55 @@ Error BitcodeReader::parseFunctionBody(Function *F) { InstructionList.push_back(I); break; } - case bitc::FUNC_CODE_INST_CMPXCHG_OLD: - case bitc::FUNC_CODE_INST_CMPXCHG: { - // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, ssid, - // failureordering?, isweak?] + case bitc::FUNC_CODE_INST_CMPXCHG_OLD: { + // CMPXCHG_OLD: [ptrty, ptr, cmp, val, vol, ordering, synchscope, + // failure_ordering?, weak?] + const size_t NumRecords = Record.size(); unsigned OpNum = 0; - Value *Ptr, *Cmp, *New; + Value *Ptr = nullptr; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, &FullTy)) return error("Invalid record"); if (!isa<PointerType>(Ptr->getType())) return error("Cmpxchg operand is not a pointer type"); - if (BitCode == bitc::FUNC_CODE_INST_CMPXCHG) { - if (getValueTypePair(Record, OpNum, NextValueNo, Cmp, &FullTy)) - return error("Invalid record"); - } else if (popValue(Record, OpNum, NextValueNo, - getPointerElementFlatType(FullTy), Cmp)) + Value *Cmp = nullptr; + if (popValue(Record, OpNum, NextValueNo, + getPointerElementFlatType(FullTy), Cmp)) return error("Invalid record"); - else - FullTy = cast<PointerType>(FullTy)->getElementType(); + FullTy = cast<PointerType>(FullTy)->getElementType(); + + Value *New = nullptr; if (popValue(Record, OpNum, NextValueNo, Cmp->getType(), New) || - Record.size() < OpNum + 3 || Record.size() > OpNum + 5) + NumRecords < OpNum + 3 || NumRecords > OpNum + 5) return error("Invalid record"); - AtomicOrdering SuccessOrdering = getDecodedOrdering(Record[OpNum + 1]); + const AtomicOrdering SuccessOrdering = + getDecodedOrdering(Record[OpNum + 1]); if (SuccessOrdering == AtomicOrdering::NotAtomic || SuccessOrdering == AtomicOrdering::Unordered) return error("Invalid record"); - SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 2]); + + const SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 2]); if (Error Err = typeCheckLoadStoreInst(Cmp->getType(), Ptr->getType())) return Err; - AtomicOrdering FailureOrdering; - if (Record.size() < 7) - FailureOrdering = - AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering); - else - FailureOrdering = getDecodedOrdering(Record[OpNum + 3]); - Align Alignment( + const AtomicOrdering FailureOrdering = + NumRecords < 7 + ? AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering) + : getDecodedOrdering(Record[OpNum + 3]); + + const Align Alignment( TheModule->getDataLayout().getTypeStoreSize(Cmp->getType())); + I = new AtomicCmpXchgInst(Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID); - FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)}); cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]); + FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)}); - if (Record.size() < 8) { + if (NumRecords < 8) { // Before weak cmpxchgs existed, the instruction simply returned the // value loaded from memory, so bitcode files from that era will be // expecting the first component of a modern cmpxchg. @@ -5037,12 +5072,59 @@ Error BitcodeReader::parseFunctionBody(Function *F) { I = ExtractValueInst::Create(I, 0); FullTy = cast<StructType>(FullTy)->getElementType(0); } else { - cast<AtomicCmpXchgInst>(I)->setWeak(Record[OpNum+4]); + cast<AtomicCmpXchgInst>(I)->setWeak(Record[OpNum + 4]); } InstructionList.push_back(I); break; } + case bitc::FUNC_CODE_INST_CMPXCHG: { + // CMPXCHG: [ptrty, ptr, cmp, val, vol, success_ordering, synchscope, + // failure_ordering, weak] + const size_t NumRecords = Record.size(); + unsigned OpNum = 0; + Value *Ptr = nullptr; + if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, &FullTy)) + return error("Invalid record"); + + if (!isa<PointerType>(Ptr->getType())) + return error("Cmpxchg operand is not a pointer type"); + + Value *Cmp = nullptr; + if (getValueTypePair(Record, OpNum, NextValueNo, Cmp, &FullTy)) + return error("Invalid record"); + + Value *Val = nullptr; + if (popValue(Record, OpNum, NextValueNo, Cmp->getType(), Val) || + NumRecords < OpNum + 3 || NumRecords > OpNum + 5) + return error("Invalid record"); + + const AtomicOrdering SuccessOrdering = + getDecodedOrdering(Record[OpNum + 1]); + if (SuccessOrdering == AtomicOrdering::NotAtomic || + SuccessOrdering == AtomicOrdering::Unordered) + return error("Invalid record"); + + const SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 2]); + + if (Error Err = typeCheckLoadStoreInst(Cmp->getType(), Ptr->getType())) + return Err; + + const AtomicOrdering FailureOrdering = + getDecodedOrdering(Record[OpNum + 3]); + + const Align Alignment( + TheModule->getDataLayout().getTypeStoreSize(Cmp->getType())); + + I = new AtomicCmpXchgInst(Ptr, Cmp, Val, Alignment, SuccessOrdering, + FailureOrdering, SSID); + FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)}); + cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]); + cast<AtomicCmpXchgInst>(I)->setWeak(Record[OpNum + 4]); + + InstructionList.push_back(I); + break; + } case bitc::FUNC_CODE_INST_ATOMICRMW: { // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, ssid] unsigned OpNum = 0; @@ -5172,7 +5254,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { TCK = CallInst::TCK_NoTail; cast<CallInst>(I)->setTailCallKind(TCK); cast<CallInst>(I)->setAttributes(PAL); - propagateByValTypes(cast<CallBase>(I), ArgsFullTys); + propagateByValSRetTypes(cast<CallBase>(I), ArgsFullTys); if (FMF.any()) { if (!isa<FPMathOperator>(I)) return error("Fast-math-flags specified for call without " @@ -5200,7 +5282,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // number of operand bundle blocks. These blocks are read into // OperandBundles and consumed at the next call or invoke instruction. - if (Record.size() < 1 || Record[0] >= BundleTags.size()) + if (Record.empty() || Record[0] >= BundleTags.size()) return error("Invalid record"); std::vector<Value *> Inputs; @@ -5390,6 +5472,36 @@ Error BitcodeReader::materialize(GlobalValue *GV) { } } + // "Upgrade" older incorrect branch weights by dropping them. + for (auto &I : instructions(F)) { + if (auto *MD = I.getMetadata(LLVMContext::MD_prof)) { + if (MD->getOperand(0) != nullptr && isa<MDString>(MD->getOperand(0))) { + MDString *MDS = cast<MDString>(MD->getOperand(0)); + StringRef ProfName = MDS->getString(); + // Check consistency of !prof branch_weights metadata. + if (!ProfName.equals("branch_weights")) + continue; + unsigned ExpectedNumOperands = 0; + if (BranchInst *BI = dyn_cast<BranchInst>(&I)) + ExpectedNumOperands = BI->getNumSuccessors(); + else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I)) + ExpectedNumOperands = SI->getNumSuccessors(); + else if (isa<CallInst>(&I)) + ExpectedNumOperands = 1; + else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I)) + ExpectedNumOperands = IBI->getNumDestinations(); + else if (isa<SelectInst>(&I)) + ExpectedNumOperands = 2; + else + continue; // ignore and continue. + + // If branch weight doesn't match, just strip branch weight. + if (MD->getNumOperands() != 1 + ExpectedNumOperands) + I.setMetadata(LLVMContext::MD_prof, nullptr); + } + } + } + // Look for functions that rely on old function attribute behavior. UpgradeFunctionAttributes(*F); @@ -5703,7 +5815,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { } /// MODULE_CODE_VSTOFFSET: [offset] case bitc::MODULE_CODE_VSTOFFSET: - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); // Note that we subtract 1 here because the offset is relative to one // word before the start of the identification or module block, which @@ -5821,8 +5933,8 @@ static void parseTypeIdSummaryRecord(ArrayRef<uint64_t> Record, parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId); } -static std::vector<FunctionSummary::ParamAccess> -parseParamAccesses(ArrayRef<uint64_t> Record) { +std::vector<FunctionSummary::ParamAccess> +ModuleSummaryIndexBitcodeReader::parseParamAccesses(ArrayRef<uint64_t> Record) { auto ReadRange = [&]() { APInt Lower(FunctionSummary::ParamAccess::RangeWidth, BitcodeReader::decodeSignRotatedValue(Record.front())); @@ -5848,7 +5960,7 @@ parseParamAccesses(ArrayRef<uint64_t> Record) { for (auto &Call : ParamAccess.Calls) { Call.ParamNo = Record.front(); Record = Record.drop_front(); - Call.Callee = Record.front(); + Call.Callee = getValueInfoFromValueId(Record.front()).first; Record = Record.drop_front(); Call.Offsets = ReadRange(); } @@ -6245,8 +6357,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { } case bitc::FS_TYPE_TESTS: assert(PendingTypeTests.empty()); - PendingTypeTests.insert(PendingTypeTests.end(), Record.begin(), - Record.end()); + llvm::append_range(PendingTypeTests, Record); break; case bitc::FS_TYPE_TEST_ASSUME_VCALLS: diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp index a8bf579bd180..8cdda62870ff 100644 --- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp @@ -17,7 +17,6 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" @@ -63,7 +62,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBuffer.h" @@ -75,7 +73,6 @@ #include <deque> #include <limits> #include <map> -#include <memory> #include <string> #include <system_error> #include <tuple> @@ -368,7 +365,7 @@ public: ~PlaceholderQueue() { assert(empty() && "PlaceholderQueue hasn't been flushed before being destroyed"); } - bool empty() { return PHs.empty(); } + bool empty() const { return PHs.empty(); } DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID); void flush(BitcodeReaderMetadataList &MetadataList); @@ -441,6 +438,20 @@ class MetadataLoader::MetadataLoaderImpl { /// Index that keeps track of where to find a metadata record in the stream. std::vector<uint64_t> GlobalMetadataBitPosIndex; + /// Cursor position of the start of the global decl attachments, to enable + /// loading using the index built for lazy loading, instead of forward + /// references. + uint64_t GlobalDeclAttachmentPos = 0; + +#ifndef NDEBUG + /// Sanity check that we end up parsing all of the global decl attachments. + unsigned NumGlobalDeclAttachSkipped = 0; + unsigned NumGlobalDeclAttachParsed = 0; +#endif + + /// Load the global decl attachments, using the index built for lazy loading. + Expected<bool> loadGlobalDeclAttachments(); + /// Populate the index above to enable lazily loading of metadata, and load /// the named metadata as well as the transitively referenced global /// Metadata. @@ -665,7 +676,7 @@ public: return FunctionsWithSPs.lookup(F); } - bool hasSeenOldLoopTags() { return HasSeenOldLoopTags; } + bool hasSeenOldLoopTags() const { return HasSeenOldLoopTags; } Error parseMetadataAttachment( Function &F, const SmallVectorImpl<Instruction *> &InstructionList); @@ -673,7 +684,7 @@ public: Error parseMetadataKinds(); void setStripTBAA(bool Value) { StripTBAA = Value; } - bool isStrippingTBAA() { return StripTBAA; } + bool isStrippingTBAA() const { return StripTBAA; } unsigned size() const { return MetadataList.size(); } void shrinkTo(unsigned N) { MetadataList.shrinkTo(N); } @@ -684,8 +695,10 @@ Expected<bool> MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { IndexCursor = Stream; SmallVector<uint64_t, 64> Record; + GlobalDeclAttachmentPos = 0; // Get the abbrevs, and preload record positions to make them lazy-loadable. while (true) { + uint64_t SavedPos = IndexCursor.GetCurrentBitNo(); Expected<BitstreamEntry> MaybeEntry = IndexCursor.advanceSkippingSubblocks( BitstreamCursor::AF_DontPopBlockAtEnd); if (!MaybeEntry) @@ -820,25 +833,11 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { break; } case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { - // FIXME: we need to do this early because we don't materialize global - // value explicitly. - if (Error Err = IndexCursor.JumpToBit(CurrentPos)) - return std::move(Err); - Record.clear(); - if (Expected<unsigned> MaybeRecord = - IndexCursor.readRecord(Entry.ID, Record)) - ; - else - return MaybeRecord.takeError(); - if (Record.size() % 2 == 0) - return error("Invalid record"); - unsigned ValueID = Record[0]; - if (ValueID >= ValueList.size()) - return error("Invalid record"); - if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID])) - if (Error Err = parseGlobalObjectAttachment( - *GO, ArrayRef<uint64_t>(Record).slice(1))) - return std::move(Err); + if (!GlobalDeclAttachmentPos) + GlobalDeclAttachmentPos = SavedPos; +#ifndef NDEBUG + NumGlobalDeclAttachSkipped++; +#endif break; } case bitc::METADATA_KIND: @@ -853,6 +852,7 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { case bitc::METADATA_SUBRANGE: case bitc::METADATA_ENUMERATOR: case bitc::METADATA_BASIC_TYPE: + case bitc::METADATA_STRING_TYPE: case bitc::METADATA_DERIVED_TYPE: case bitc::METADATA_COMPOSITE_TYPE: case bitc::METADATA_SUBROUTINE_TYPE: @@ -875,6 +875,7 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { case bitc::METADATA_OBJC_PROPERTY: case bitc::METADATA_IMPORTED_ENTITY: case bitc::METADATA_GLOBAL_VAR_EXPR: + case bitc::METADATA_GENERIC_SUBRANGE: // We don't expect to see any of these, if we see one, give up on // lazy-loading and fallback. MDStringRef.clear(); @@ -887,6 +888,83 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { } } +// Load the global decl attachments after building the lazy loading index. +// We don't load them "lazily" - all global decl attachments must be +// parsed since they aren't materialized on demand. However, by delaying +// their parsing until after the index is created, we can use the index +// instead of creating temporaries. +Expected<bool> MetadataLoader::MetadataLoaderImpl::loadGlobalDeclAttachments() { + // Nothing to do if we didn't find any of these metadata records. + if (!GlobalDeclAttachmentPos) + return true; + // Use a temporary cursor so that we don't mess up the main Stream cursor or + // the lazy loading IndexCursor (which holds the necessary abbrev ids). + BitstreamCursor TempCursor = Stream; + SmallVector<uint64_t, 64> Record; + // Jump to the position before the first global decl attachment, so we can + // scan for the first BitstreamEntry record. + if (Error Err = TempCursor.JumpToBit(GlobalDeclAttachmentPos)) + return std::move(Err); + while (true) { + Expected<BitstreamEntry> MaybeEntry = TempCursor.advanceSkippingSubblocks( + BitstreamCursor::AF_DontPopBlockAtEnd); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + // Sanity check that we parsed them all. + assert(NumGlobalDeclAttachSkipped == NumGlobalDeclAttachParsed); + return true; + case BitstreamEntry::Record: + break; + } + uint64_t CurrentPos = TempCursor.GetCurrentBitNo(); + Expected<unsigned> MaybeCode = TempCursor.skipRecord(Entry.ID); + if (!MaybeCode) + return MaybeCode.takeError(); + if (MaybeCode.get() != bitc::METADATA_GLOBAL_DECL_ATTACHMENT) { + // Anything other than a global decl attachment signals the end of + // these records. sanity check that we parsed them all. + assert(NumGlobalDeclAttachSkipped == NumGlobalDeclAttachParsed); + return true; + } +#ifndef NDEBUG + NumGlobalDeclAttachParsed++; +#endif + // FIXME: we need to do this early because we don't materialize global + // value explicitly. + if (Error Err = TempCursor.JumpToBit(CurrentPos)) + return std::move(Err); + Record.clear(); + if (Expected<unsigned> MaybeRecord = + TempCursor.readRecord(Entry.ID, Record)) + ; + else + return MaybeRecord.takeError(); + if (Record.size() % 2 == 0) + return error("Invalid record"); + unsigned ValueID = Record[0]; + if (ValueID >= ValueList.size()) + return error("Invalid record"); + if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID])) { + // Need to save and restore the current position since + // parseGlobalObjectAttachment will resolve all forward references which + // would require parsing from locations stored in the index. + CurrentPos = TempCursor.GetCurrentBitNo(); + if (Error Err = parseGlobalObjectAttachment( + *GO, ArrayRef<uint64_t>(Record).slice(1))) + return std::move(Err); + if (Error Err = TempCursor.JumpToBit(CurrentPos)) + return std::move(Err); + } + } +} + /// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing /// module level metadata. Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { @@ -916,6 +994,14 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { MetadataList.resize(MDStringRef.size() + GlobalMetadataBitPosIndex.size()); + // Now that we have built the index, load the global decl attachments + // that were deferred during that process. This avoids creating + // temporaries. + SuccessOrErr = loadGlobalDeclAttachments(); + if (!SuccessOrErr) + return SuccessOrErr.takeError(); + assert(SuccessOrErr.get()); + // Reading the named metadata created forward references and/or // placeholders, that we flush here. resolveForwardRefsAndPlaceholders(Placeholders); @@ -1286,6 +1372,18 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( NextMetadataNo++; break; } + case bitc::METADATA_GENERIC_SUBRANGE: { + Metadata *Val = nullptr; + Val = GET_OR_DISTINCT(DIGenericSubrange, + (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]), getMDOrNull(Record[3]), + getMDOrNull(Record[4]))); + + MetadataList.assignValue(Val, NextMetadataNo); + IsDistinct = Record[0] & 1; + NextMetadataNo++; + break; + } case bitc::METADATA_ENUMERATOR: { if (Record.size() < 3) return error("Invalid record"); @@ -1325,6 +1423,20 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( NextMetadataNo++; break; } + case bitc::METADATA_STRING_TYPE: { + if (Record.size() != 8) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIStringType, + (Context, Record[1], getMDString(Record[2]), + getMDOrNull(Record[3]), getMDOrNull(Record[4]), + Record[5], Record[6], Record[7])), + NextMetadataNo); + NextMetadataNo++; + break; + } case bitc::METADATA_DERIVED_TYPE: { if (Record.size() < 12 || Record.size() > 13) return error("Invalid record"); @@ -1350,7 +1462,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_COMPOSITE_TYPE: { - if (Record.size() < 16 || Record.size() > 18) + if (Record.size() < 16 || Record.size() > 21) return error("Invalid record"); // If we have a UUID and this is not a forward declaration, lookup the @@ -1375,6 +1487,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Metadata *TemplateParams = nullptr; Metadata *Discriminator = nullptr; Metadata *DataLocation = nullptr; + Metadata *Associated = nullptr; + Metadata *Allocated = nullptr; + Metadata *Rank = nullptr; auto *Identifier = getMDString(Record[15]); // If this module is being parsed so that it can be ThinLTO imported // into another module, composite types only need to be imported @@ -1399,13 +1514,21 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Discriminator = getMDOrNull(Record[16]); if (Record.size() > 17) DataLocation = getMDOrNull(Record[17]); + if (Record.size() > 19) { + Associated = getMDOrNull(Record[18]); + Allocated = getMDOrNull(Record[19]); + } + if (Record.size() > 20) { + Rank = getMDOrNull(Record[20]); + } } DICompositeType *CT = nullptr; if (Identifier) CT = DICompositeType::buildODRType( Context, *Identifier, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams, Discriminator, DataLocation); + VTableHolder, TemplateParams, Discriminator, DataLocation, Associated, + Allocated, Rank); // Create a node if we didn't get a lazy ODR type. if (!CT) @@ -1413,7 +1536,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( (Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams, - Identifier, Discriminator, DataLocation)); + Identifier, Discriminator, DataLocation, Associated, + Allocated, Rank)); if (!IsNotUsedInTypeRef && Identifier) MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT)); @@ -1441,19 +1565,20 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( } case bitc::METADATA_MODULE: { - if (Record.size() < 5 || Record.size() > 8) + if (Record.size() < 5 || Record.size() > 9) return error("Invalid record"); - unsigned Offset = Record.size() >= 7 ? 2 : 1; + unsigned Offset = Record.size() >= 8 ? 2 : 1; IsDistinct = Record[0]; MetadataList.assignValue( GET_OR_DISTINCT( DIModule, - (Context, Record.size() >= 7 ? getMDOrNull(Record[1]) : nullptr, + (Context, Record.size() >= 8 ? getMDOrNull(Record[1]) : nullptr, getMDOrNull(Record[0 + Offset]), getMDString(Record[1 + Offset]), getMDString(Record[2 + Offset]), getMDString(Record[3 + Offset]), getMDString(Record[4 + Offset]), - Record.size() <= 7 ? 0 : Record[7])), + Record.size() <= 7 ? 0 : Record[7], + Record.size() <= 8 ? false : Record[8])), NextMetadataNo); NextMetadataNo++; break; @@ -2001,7 +2126,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseGlobalObjectAttachment( auto K = MDKindMap.find(Record[I]); if (K == MDKindMap.end()) return error("Invalid ID"); - MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[I + 1]); + MDNode *MD = + dyn_cast_or_null<MDNode>(getMetadataFwdRefOrLoad(Record[I + 1])); if (!MD) return error("Invalid metadata attachment: expect fwd ref to MDNode"); GO.addMetadata(K->second, *MD); diff --git a/llvm/lib/Bitcode/Reader/ValueList.cpp b/llvm/lib/Bitcode/Reader/ValueList.cpp index 63a206eeb022..ddfa28c6b1e4 100644 --- a/llvm/lib/Bitcode/Reader/ValueList.cpp +++ b/llvm/lib/Bitcode/Reader/ValueList.cpp @@ -16,14 +16,11 @@ #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" -#include "llvm/IR/ValueHandle.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include <algorithm> -#include <cassert> #include <cstddef> #include <limits> -#include <utility> using namespace llvm; |