diff options
Diffstat (limited to 'lib/Bitcode')
-rw-r--r-- | lib/Bitcode/Reader/BitcodeReader.cpp | 193 | ||||
-rw-r--r-- | lib/Bitcode/Reader/MetadataLoader.cpp | 74 | ||||
-rw-r--r-- | lib/Bitcode/Reader/ValueList.cpp | 6 | ||||
-rw-r--r-- | lib/Bitcode/Writer/BitWriter.cpp | 6 | ||||
-rw-r--r-- | lib/Bitcode/Writer/BitcodeWriter.cpp | 228 | ||||
-rw-r--r-- | lib/Bitcode/Writer/BitcodeWriterPass.cpp | 8 | ||||
-rw-r--r-- | lib/Bitcode/Writer/ValueEnumerator.cpp | 7 |
7 files changed, 407 insertions, 115 deletions
diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 95291a1caf9a..c45b441238bc 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Config/llvm-config.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/AutoUpgrade.h" @@ -532,7 +533,7 @@ public: Error materializeModule() override; std::vector<StructType *> getIdentifiedStructTypes() const override; - /// \brief Main interface to parsing a bitcode buffer. + /// Main interface to parsing a bitcode buffer. /// \returns true if an error occurred. Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata = false, bool IsImporting = false); @@ -743,14 +744,16 @@ private: std::vector<ValueInfo> makeRefList(ArrayRef<uint64_t> Record); std::vector<FunctionSummary::EdgeTy> makeCallList(ArrayRef<uint64_t> Record, bool IsOldProfileFormat, - bool HasProfile); + bool HasProfile, + bool HasRelBF); Error parseEntireSummary(unsigned ID); Error parseModuleStringTable(); std::pair<ValueInfo, GlobalValue::GUID> getValueInfoFromValueId(unsigned ValueId); - ModuleSummaryIndex::ModuleInfo *addThisModule(); + void addThisModule(); + ModuleSummaryIndex::ModuleInfo *getThisModule(); }; } // end anonymous namespace @@ -1046,19 +1049,21 @@ static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) { static FastMathFlags getDecodedFastMathFlags(unsigned Val) { FastMathFlags FMF; - if (0 != (Val & FastMathFlags::AllowReassoc)) + if (0 != (Val & bitc::UnsafeAlgebra)) + FMF.setFast(); + if (0 != (Val & bitc::AllowReassoc)) FMF.setAllowReassoc(); - if (0 != (Val & FastMathFlags::NoNaNs)) + if (0 != (Val & bitc::NoNaNs)) FMF.setNoNaNs(); - if (0 != (Val & FastMathFlags::NoInfs)) + if (0 != (Val & bitc::NoInfs)) FMF.setNoInfs(); - if (0 != (Val & FastMathFlags::NoSignedZeros)) + if (0 != (Val & bitc::NoSignedZeros)) FMF.setNoSignedZeros(); - if (0 != (Val & FastMathFlags::AllowReciprocal)) + if (0 != (Val & bitc::AllowReciprocal)) FMF.setAllowReciprocal(); - if (0 != (Val & FastMathFlags::AllowContract)) + if (0 != (Val & bitc::AllowContract)) FMF.setAllowContract(true); - if (0 != (Val & FastMathFlags::ApproxFunc)) + if (0 != (Val & bitc::ApproxFunc)) FMF.setApproxFunc(); return FMF; } @@ -1157,6 +1162,9 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { case Attribute::Speculatable: return 1ULL << 54; case Attribute::StrictFP: return 1ULL << 55; case Attribute::SanitizeHWAddress: return 1ULL << 56; + case Attribute::NoCfCheck: return 1ULL << 57; + case Attribute::OptForFuzzing: return 1ULL << 58; + case Attribute::ShadowCallStack: return 1ULL << 59; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1195,7 +1203,7 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) { } } -/// \brief This fills an AttrBuilder object with the LLVM attributes that have +/// This fills an AttrBuilder object with the LLVM attributes that have /// been decoded from the given integer. This function must stay in sync with /// 'encodeLLVMAttributesForBitcode'. static void decodeLLVMAttributesForBitcode(AttrBuilder &B, @@ -1335,8 +1343,12 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoRedZone; case bitc::ATTR_KIND_NO_RETURN: return Attribute::NoReturn; + case bitc::ATTR_KIND_NOCF_CHECK: + return Attribute::NoCfCheck; case bitc::ATTR_KIND_NO_UNWIND: return Attribute::NoUnwind; + case bitc::ATTR_KIND_OPT_FOR_FUZZING: + return Attribute::OptForFuzzing; case bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE: return Attribute::OptimizeForSize; case bitc::ATTR_KIND_OPTIMIZE_NONE: @@ -1363,6 +1375,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::StackProtectStrong; case bitc::ATTR_KIND_SAFESTACK: return Attribute::SafeStack; + case bitc::ATTR_KIND_SHADOWCALLSTACK: + return Attribute::ShadowCallStack; case bitc::ATTR_KIND_STRICT_FP: return Attribute::StrictFP; case bitc::ATTR_KIND_STRUCT_RET: @@ -2510,6 +2524,7 @@ Error BitcodeReader::parseConstants() { for (unsigned i = 0; i != ConstStrSize; ++i) ConstrStr += (char)Record[3+AsmStrSize+i]; PointerType *PTy = cast<PointerType>(CurTy); + UpgradeInlineAsmString(&AsmStr); V = InlineAsm::get(cast<FunctionType>(PTy->getElementType()), AsmStr, ConstrStr, HasSideEffects, IsAlignStack); break; @@ -2535,6 +2550,7 @@ Error BitcodeReader::parseConstants() { for (unsigned i = 0; i != ConstStrSize; ++i) ConstrStr += (char)Record[3+AsmStrSize+i]; PointerType *PTy = cast<PointerType>(CurTy); + UpgradeInlineAsmString(&AsmStr); V = InlineAsm::get(cast<FunctionType>(PTy->getElementType()), AsmStr, ConstrStr, HasSideEffects, IsAlignStack, InlineAsm::AsmDialect(AsmDialect)); @@ -2815,6 +2831,13 @@ Error BitcodeReader::parseComdatRecord(ArrayRef<uint64_t> Record) { return Error::success(); } +static void inferDSOLocal(GlobalValue *GV) { + // infer dso_local from linkage and visibility if it is not encoded. + if (GV->hasLocalLinkage() || + (!GV->hasDefaultVisibility() && !GV->hasExternalWeakLinkage())) + GV->setDSOLocal(true); +} + Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) { // v1: [pointer type, isconst, initid, linkage, alignment, section, // visibility, threadlocal, unnamed_addr, externally_initialized, @@ -2907,6 +2930,7 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) { if (Record.size() > 13) { NewGV->setDSOLocal(getDecodedDSOLocal(Record[13])); } + inferDSOLocal(NewGV); return Error::success(); } @@ -2991,6 +3015,7 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) { if (Record.size() > 15) { Func->setDSOLocal(getDecodedDSOLocal(Record[15])); } + inferDSOLocal(Func); ValueList.push_back(Func); @@ -3054,16 +3079,21 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( // FIXME: Change to an error if non-default in 4.0. NewGA->setVisibility(getDecodedVisibility(Record[VisInd])); } - if (OpNum != Record.size()) - NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++])); - else - upgradeDLLImportExportLinkage(NewGA, Linkage); - if (OpNum != Record.size()) - NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++])); - if (OpNum != Record.size()) - NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++])); + if (BitCode == bitc::MODULE_CODE_ALIAS || + BitCode == bitc::MODULE_CODE_ALIAS_OLD) { + if (OpNum != Record.size()) + NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++])); + else + upgradeDLLImportExportLinkage(NewGA, Linkage); + if (OpNum != Record.size()) + NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++])); + if (OpNum != Record.size()) + NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++])); + } if (OpNum != Record.size()) NewGA->setDSOLocal(getDecodedDSOLocal(Record[OpNum++])); + inferDSOLocal(NewGA); + ValueList.push_back(NewGA); IndirectSymbolInits.push_back(std::make_pair(NewGA, Val)); return Error::success(); @@ -4773,6 +4803,9 @@ Error BitcodeReader::materializeModule() { UpgradeDebugInfo(*TheModule); UpgradeModuleFlags(*TheModule); + + UpgradeRetainReleaseMarker(*TheModule); + return Error::success(); } @@ -4786,9 +4819,13 @@ ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader( : BitcodeReaderBase(std::move(Cursor), Strtab), TheIndex(TheIndex), ModulePath(ModulePath), ModuleId(ModuleId) {} +void ModuleSummaryIndexBitcodeReader::addThisModule() { + TheIndex.addModule(ModulePath, ModuleId); +} + ModuleSummaryIndex::ModuleInfo * -ModuleSummaryIndexBitcodeReader::addThisModule() { - return TheIndex.addModule(ModulePath, ModuleId); +ModuleSummaryIndexBitcodeReader::getThisModule() { + return TheIndex.getModule(ModulePath); } std::pair<ValueInfo, GlobalValue::GUID> @@ -4810,8 +4847,15 @@ void ModuleSummaryIndexBitcodeReader::setValueGUID( if (PrintSummaryGUIDs) dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is " << ValueName << "\n"; - ValueIdToValueInfoMap[ValueID] = - std::make_pair(TheIndex.getOrInsertValueInfo(ValueGUID), OriginalNameID); + + // UseStrtab is false for legacy summary formats and value names are + // created on stack. In that case we save the name in a string saver in + // the index so that the value name can be recorded. + ValueIdToValueInfoMap[ValueID] = std::make_pair( + TheIndex.getOrInsertValueInfo( + ValueGUID, + UseStrtab ? ValueName : TheIndex.saveString(ValueName.str())), + OriginalNameID); } // Specialized value symbol table parser used when reading module index @@ -4940,6 +4984,9 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { break; case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: case bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID: + // Add the module if it is a per-module index (has a source file name). + if (!SourceFileName.empty()) + addThisModule(); assert(!SeenValueSymbolTable && "Already read VST when parsing summary block?"); // We might not have a VST if there were no values in the @@ -4985,7 +5032,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { case bitc::MODULE_CODE_HASH: { if (Record.size() != 5) return error("Invalid hash length " + Twine(Record.size()).str()); - auto &Hash = addThisModule()->second.second; + auto &Hash = getThisModule()->second.second; int Pos = 0; for (auto &Val : Record) { assert(!(Val >> 32) && "Unexpected high bits set"); @@ -5040,12 +5087,15 @@ ModuleSummaryIndexBitcodeReader::makeRefList(ArrayRef<uint64_t> Record) { return Ret; } -std::vector<FunctionSummary::EdgeTy> ModuleSummaryIndexBitcodeReader::makeCallList( - ArrayRef<uint64_t> Record, bool IsOldProfileFormat, bool HasProfile) { +std::vector<FunctionSummary::EdgeTy> +ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef<uint64_t> Record, + bool IsOldProfileFormat, + bool HasProfile, bool HasRelBF) { std::vector<FunctionSummary::EdgeTy> Ret; Ret.reserve(Record.size()); for (unsigned I = 0, E = Record.size(); I != E; ++I) { CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown; + uint64_t RelBF = 0; ValueInfo Callee = getValueInfoFromValueId(Record[I]).first; if (IsOldProfileFormat) { I += 1; // Skip old callsitecount field @@ -5053,11 +5103,63 @@ std::vector<FunctionSummary::EdgeTy> ModuleSummaryIndexBitcodeReader::makeCallLi I += 1; // Skip old profilecount field } else if (HasProfile) Hotness = static_cast<CalleeInfo::HotnessType>(Record[++I]); - Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}}); + else if (HasRelBF) + RelBF = Record[++I]; + Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo(Hotness, RelBF)}); } return Ret; } +static void +parseWholeProgramDevirtResolutionByArg(ArrayRef<uint64_t> Record, size_t &Slot, + WholeProgramDevirtResolution &Wpd) { + uint64_t ArgNum = Record[Slot++]; + WholeProgramDevirtResolution::ByArg &B = + Wpd.ResByArg[{Record.begin() + Slot, Record.begin() + Slot + ArgNum}]; + Slot += ArgNum; + + B.TheKind = + static_cast<WholeProgramDevirtResolution::ByArg::Kind>(Record[Slot++]); + B.Info = Record[Slot++]; + B.Byte = Record[Slot++]; + B.Bit = Record[Slot++]; +} + +static void parseWholeProgramDevirtResolution(ArrayRef<uint64_t> Record, + StringRef Strtab, size_t &Slot, + TypeIdSummary &TypeId) { + uint64_t Id = Record[Slot++]; + WholeProgramDevirtResolution &Wpd = TypeId.WPDRes[Id]; + + Wpd.TheKind = static_cast<WholeProgramDevirtResolution::Kind>(Record[Slot++]); + Wpd.SingleImplName = {Strtab.data() + Record[Slot], + static_cast<size_t>(Record[Slot + 1])}; + Slot += 2; + + uint64_t ResByArgNum = Record[Slot++]; + for (uint64_t I = 0; I != ResByArgNum; ++I) + parseWholeProgramDevirtResolutionByArg(Record, Slot, Wpd); +} + +static void parseTypeIdSummaryRecord(ArrayRef<uint64_t> Record, + StringRef Strtab, + ModuleSummaryIndex &TheIndex) { + size_t Slot = 0; + TypeIdSummary &TypeId = TheIndex.getOrInsertTypeIdSummary( + {Strtab.data() + Record[Slot], static_cast<size_t>(Record[Slot + 1])}); + Slot += 2; + + TypeId.TTRes.TheKind = static_cast<TypeTestResolution::Kind>(Record[Slot++]); + TypeId.TTRes.SizeM1BitWidth = Record[Slot++]; + TypeId.TTRes.AlignLog2 = Record[Slot++]; + TypeId.TTRes.SizeM1 = Record[Slot++]; + TypeId.TTRes.BitMask = Record[Slot++]; + TypeId.TTRes.InlineBits = Record[Slot++]; + + while (Slot < Record.size()) + parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId); +} + // Eagerly parse the entire summary block. This populates the GlobalValueSummary // objects in the index. Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { @@ -5120,6 +5222,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { switch (BitCode) { default: // Default behavior: ignore. break; + case bitc::FS_FLAGS: { // [flags] + uint64_t Flags = Record[0]; + // Scan flags (set only on the combined index). + assert(Flags <= 0x3 && "Unexpected bits in flag"); + + // 1 bit: WithGlobalValueDeadStripping flag. + if (Flags & 0x1) + TheIndex.setWithGlobalValueDeadStripping(); + // 1 bit: SkipModuleByDistributedBackend flag. + if (Flags & 0x2) + TheIndex.setSkipModuleByDistributedBackend(); + break; + } case bitc::FS_VALUE_GUID: { // [valueid, refguid] uint64_t ValueID = Record[0]; GlobalValue::GUID RefGUID = Record[1]; @@ -5132,7 +5247,11 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { // FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, // n x (valueid, hotness)] + // FS_PERMODULE_RELBF: [valueid, flags, instcount, fflags, numrefs, + // numrefs x valueid, + // n x (valueid, relblockfreq)] case bitc::FS_PERMODULE: + case bitc::FS_PERMODULE_RELBF: case bitc::FS_PERMODULE_PROFILE: { unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; @@ -5158,9 +5277,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { std::vector<ValueInfo> Refs = makeRefList( ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs)); bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE); + bool HasRelBF = (BitCode == bitc::FS_PERMODULE_RELBF); std::vector<FunctionSummary::EdgeTy> Calls = makeCallList( ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex), - IsOldProfileFormat, HasProfile); + IsOldProfileFormat, HasProfile, HasRelBF); auto FS = llvm::make_unique<FunctionSummary>( Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), std::move(Calls), std::move(PendingTypeTests), @@ -5174,7 +5294,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { PendingTypeTestAssumeConstVCalls.clear(); PendingTypeCheckedLoadConstVCalls.clear(); auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID); - FS->setModulePath(addThisModule()->first()); + FS->setModulePath(getThisModule()->first()); FS->setOriginalName(VIAndOriginalGUID.second); TheIndex.addGlobalValueSummary(VIAndOriginalGUID.first, std::move(FS)); break; @@ -5193,7 +5313,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. - AS->setModulePath(addThisModule()->first()); + AS->setModulePath(getThisModule()->first()); GlobalValue::GUID AliaseeGUID = getValueInfoFromValueId(AliaseeID).first.getGUID(); @@ -5217,7 +5337,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { std::vector<ValueInfo> Refs = makeRefList(ArrayRef<uint64_t>(Record).slice(2)); auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs)); - FS->setModulePath(addThisModule()->first()); + FS->setModulePath(getThisModule()->first()); auto GUID = getValueInfoFromValueId(ValueID); FS->setOriginalName(GUID.second); TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); @@ -5252,7 +5372,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE); std::vector<FunctionSummary::EdgeTy> Edges = makeCallList( ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex), - IsOldProfileFormat, HasProfile); + IsOldProfileFormat, HasProfile, false); ValueInfo VI = getValueInfoFromValueId(ValueID).first; auto FS = llvm::make_unique<FunctionSummary>( Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), @@ -5360,6 +5480,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { {Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])}); break; } + case bitc::FS_CFI_FUNCTION_DECLS: { std::set<std::string> &CfiFunctionDecls = TheIndex.cfiFunctionDecls(); for (unsigned I = 0; I != Record.size(); I += 2) @@ -5367,6 +5488,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { {Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])}); break; } + + case bitc::FS_TYPE_ID: + parseTypeIdSummaryRecord(Record, Strtab, TheIndex); + break; } } llvm_unreachable("Exit infinite loop"); @@ -5602,7 +5727,7 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { } } -/// \brief Get a lazy one-at-time loading module from bitcode. +/// Get a lazy one-at-time loading module from bitcode. /// /// This isn't always used in a lazy context. In particular, it's also used by /// \a parseModule(). If this is truly lazy, then we need to eagerly pull @@ -5676,7 +5801,7 @@ Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() { BitstreamCursor Stream(Buffer); Stream.JumpToBit(ModuleBit); - auto Index = llvm::make_unique<ModuleSummaryIndex>(); + auto Index = llvm::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false); ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, *Index, ModuleIdentifier, 0); diff --git a/lib/Bitcode/Reader/MetadataLoader.cpp b/lib/Bitcode/Reader/MetadataLoader.cpp index 1b0d80d26cf5..011c41e2cecd 100644 --- a/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/lib/Bitcode/Reader/MetadataLoader.cpp @@ -165,7 +165,7 @@ public: /// necessary. Metadata *getMetadataFwdRef(unsigned Idx); - /// Return the the given metadata only if it is fully resolved. + /// Return the given metadata only if it is fully resolved. /// /// Gives the same result as \a lookup(), unless \a MDNode::isResolved() /// would give \c false. @@ -822,6 +822,7 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { case bitc::METADATA_TEMPLATE_VALUE: case bitc::METADATA_GLOBAL_VAR: case bitc::METADATA_LOCAL_VAR: + case bitc::METADATA_LABEL: case bitc::METADATA_EXPRESSION: case bitc::METADATA_OBJC_PROPERTY: case bitc::METADATA_IMPORTED_ENTITY: @@ -1174,14 +1175,25 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_SUBRANGE: { - if (Record.size() != 3) - return error("Invalid record"); + Metadata *Val = nullptr; + // Operand 'count' is interpreted as: + // - Signed integer (version 0) + // - Metadata node (version 1) + switch (Record[0] >> 1) { + case 0: + Val = GET_OR_DISTINCT(DISubrange, + (Context, Record[1], unrotateSign(Record.back()))); + break; + case 1: + Val = GET_OR_DISTINCT(DISubrange, (Context, getMDOrNull(Record[1]), + unrotateSign(Record.back()))); + break; + default: + return error("Invalid record: Unsupported version of DISubrange"); + } - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DISubrange, - (Context, Record[1], unrotateSign(Record[2]))), - NextMetadataNo); + MetadataList.assignValue(Val, NextMetadataNo); + IsDistinct = Record[0] & 1; NextMetadataNo++; break; } @@ -1189,10 +1201,11 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( if (Record.size() != 3) return error("Invalid record"); - IsDistinct = Record[0]; + IsDistinct = Record[0] & 1; + bool IsUnsigned = Record[0] & 2; MetadataList.assignValue( GET_OR_DISTINCT(DIEnumerator, (Context, unrotateSign(Record[1]), - getMDString(Record[2]))), + IsUnsigned, getMDString(Record[2]))), NextMetadataNo); NextMetadataNo++; break; @@ -1235,7 +1248,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_COMPOSITE_TYPE: { - if (Record.size() != 16) + if (Record.size() < 16 || Record.size() > 17) return error("Invalid record"); // If we have a UUID and this is not a forward declaration, lookup the @@ -1258,6 +1271,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( unsigned RuntimeLang = Record[12]; Metadata *VTableHolder = nullptr; Metadata *TemplateParams = nullptr; + Metadata *Discriminator = nullptr; auto *Identifier = getMDString(Record[15]); // If this module is being parsed so that it can be ThinLTO imported // into another module, composite types only need to be imported @@ -1278,13 +1292,15 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Elements = getMDOrNull(Record[11]); VTableHolder = getDITypeRefOrNull(Record[13]); TemplateParams = getMDOrNull(Record[14]); + if (Record.size() > 16) + Discriminator = getMDOrNull(Record[16]); } DICompositeType *CT = nullptr; if (Identifier) CT = DICompositeType::buildODRType( Context, *Identifier, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams); + VTableHolder, TemplateParams, Discriminator); // Create a node if we didn't get a lazy ODR type. if (!CT) @@ -1335,17 +1351,25 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( } case bitc::METADATA_FILE: { - if (Record.size() != 3 && Record.size() != 5) + if (Record.size() != 3 && Record.size() != 5 && Record.size() != 6) return error("Invalid record"); IsDistinct = Record[0]; + Optional<DIFile::ChecksumInfo<MDString *>> Checksum; + // The BitcodeWriter writes null bytes into Record[3:4] when the Checksum + // is not present. This matches up with the old internal representation, + // and the old encoding for CSK_None in the ChecksumKind. The new + // representation reserves the value 0 in the ChecksumKind to continue to + // encode None in a backwards-compatible way. + if (Record.size() > 4 && Record[3] && Record[4]) + Checksum.emplace(static_cast<DIFile::ChecksumKind>(Record[3]), + getMDString(Record[4])); MetadataList.assignValue( GET_OR_DISTINCT( DIFile, - (Context, getMDString(Record[1]), getMDString(Record[2]), - Record.size() == 3 ? DIFile::CSK_None - : static_cast<DIFile::ChecksumKind>(Record[3]), - Record.size() == 3 ? nullptr : getMDString(Record[4]))), + (Context, getMDString(Record[1]), getMDString(Record[2]), Checksum, + Record.size() > 5 ? Optional<MDString *>(getMDString(Record[5])) + : None)), NextMetadataNo); NextMetadataNo++; break; @@ -1415,7 +1439,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( HasUnit ? CUorFn : nullptr, // unit getMDOrNull(Record[15 + Offset]), // templateParams getMDOrNull(Record[16 + Offset]), // declaration - getMDOrNull(Record[17 + Offset]), // variables + getMDOrNull(Record[17 + Offset]), // retainedNodes HasThrownTypes ? getMDOrNull(Record[20]) : nullptr // thrownTypes )); MetadataList.assignValue(SP, NextMetadataNo); @@ -1624,6 +1648,20 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( NextMetadataNo++; break; } + case bitc::METADATA_LABEL: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0] & 1; + MetadataList.assignValue( + GET_OR_DISTINCT(DILabel, + (Context, getMDOrNull(Record[1]), + getMDString(Record[2]), + getMDOrNull(Record[3]), Record[4])), + NextMetadataNo); + NextMetadataNo++; + break; + } case bitc::METADATA_EXPRESSION: { if (Record.size() < 1) return error("Invalid record"); diff --git a/lib/Bitcode/Reader/ValueList.cpp b/lib/Bitcode/Reader/ValueList.cpp index 08bfa291098c..1ab22b5cc3d1 100644 --- a/lib/Bitcode/Reader/ValueList.cpp +++ b/lib/Bitcode/Reader/ValueList.cpp @@ -32,7 +32,7 @@ namespace llvm { namespace { -/// \brief A class for maintaining the slot number definition +/// A class for maintaining the slot number definition /// as a placeholder for the actual definition for forward constants defs. class ConstantPlaceHolder : public ConstantExpr { public: @@ -46,7 +46,7 @@ public: // allocate space for exactly one operand void *operator new(size_t s) { return User::operator new(s, 1); } - /// \brief Methods to support type inquiry through isa, cast, and dyn_cast. + /// Methods to support type inquiry through isa, cast, and dyn_cast. static bool classof(const Value *V) { return isa<ConstantExpr>(V) && cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1; @@ -144,7 +144,7 @@ Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) { void BitcodeReaderValueList::resolveConstantForwardRefs() { // Sort the values by-pointer so that they are efficient to look up with a // binary search. - std::sort(ResolveConstants.begin(), ResolveConstants.end()); + llvm::sort(ResolveConstants.begin(), ResolveConstants.end()); SmallVector<Constant *, 64> NewOps; diff --git a/lib/Bitcode/Writer/BitWriter.cpp b/lib/Bitcode/Writer/BitWriter.cpp index e0388418a3d9..763cd12aa2d7 100644 --- a/lib/Bitcode/Writer/BitWriter.cpp +++ b/lib/Bitcode/Writer/BitWriter.cpp @@ -25,7 +25,7 @@ int LLVMWriteBitcodeToFile(LLVMModuleRef M, const char *Path) { if (EC) return -1; - WriteBitcodeToFile(unwrap(M), OS); + WriteBitcodeToFile(*unwrap(M), OS); return 0; } @@ -33,7 +33,7 @@ int LLVMWriteBitcodeToFD(LLVMModuleRef M, int FD, int ShouldClose, int Unbuffered) { raw_fd_ostream OS(FD, ShouldClose, Unbuffered); - WriteBitcodeToFile(unwrap(M), OS); + WriteBitcodeToFile(*unwrap(M), OS); return 0; } @@ -45,6 +45,6 @@ LLVMMemoryBufferRef LLVMWriteBitcodeToMemoryBuffer(LLVMModuleRef M) { std::string Data; raw_string_ostream OS(Data); - WriteBitcodeToFile(unwrap(M), OS); + WriteBitcodeToFile(*unwrap(M), OS); return wrap(MemoryBuffer::getMemBufferCopy(OS.str()).release()); } diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index a7201ed97350..be75df0820d9 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -28,6 +28,7 @@ #include "llvm/Bitcode/BitCodes.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Config/llvm-config.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" @@ -86,6 +87,12 @@ static cl::opt<unsigned> cl::desc("Number of metadatas above which we emit an index " "to enable lazy-loading")); +cl::opt<bool> WriteRelBFToSummary( + "write-relbf-to-summary", cl::Hidden, cl::init(false), + cl::desc("Write relative block frequency to function summary ")); + +extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold; + namespace { /// These are manifest constants used by the bitcode writer. They do not need to @@ -167,12 +174,12 @@ protected: public: /// Constructs a ModuleBitcodeWriterBase object for the given Module, /// writing to the provided \p Buffer. - ModuleBitcodeWriterBase(const Module *M, StringTableBuilder &StrtabBuilder, + ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index) - : BitcodeWriterBase(Stream, StrtabBuilder), M(*M), - VE(*M, ShouldPreserveUseListOrder), Index(Index) { + : BitcodeWriterBase(Stream, StrtabBuilder), M(M), + VE(M, ShouldPreserveUseListOrder), Index(Index) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). @@ -190,7 +197,7 @@ public: // otherwise we would have a Value for it). If so, synthesize // a value id. for (auto &CallEdge : FS->calls()) - if (!CallEdge.first.getValue()) + if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue()) assignValueId(CallEdge.first.getGUID()); } @@ -223,7 +230,7 @@ private: // Helper to get the valueId for the type of value recorded in VI. unsigned getValueId(ValueInfo VI) { - if (!VI.getValue()) + if (!VI.haveGVs() || !VI.getValue()) return getValueId(VI.getGUID()); return VE.getValueID(VI.getValue()); } @@ -251,7 +258,7 @@ class ModuleBitcodeWriter : public ModuleBitcodeWriterBase { public: /// Constructs a ModuleBitcodeWriter object for the given Module, /// writing to the provided \p Buffer. - ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, + ModuleBitcodeWriter(const Module &M, SmallVectorImpl<char> &Buffer, StringTableBuilder &StrtabBuilder, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, bool GenerateHash, @@ -328,6 +335,8 @@ private: unsigned Abbrev); void writeDILocalVariable(const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDILabel(const DILabel *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); void writeDIExpression(const DIExpression *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N, @@ -635,8 +644,12 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NO_RED_ZONE; case Attribute::NoReturn: return bitc::ATTR_KIND_NO_RETURN; + case Attribute::NoCfCheck: + return bitc::ATTR_KIND_NOCF_CHECK; case Attribute::NoUnwind: return bitc::ATTR_KIND_NO_UNWIND; + case Attribute::OptForFuzzing: + return bitc::ATTR_KIND_OPT_FOR_FUZZING; case Attribute::OptimizeForSize: return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE; case Attribute::OptimizeNone: @@ -663,6 +676,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_STACK_PROTECT_STRONG; case Attribute::SafeStack: return bitc::ATTR_KIND_SAFESTACK; + case Attribute::ShadowCallStack: + return bitc::ATTR_KIND_SHADOWCALLSTACK; case Attribute::StrictFP: return bitc::ATTR_KIND_STRICT_FP; case Attribute::StructRet: @@ -1302,7 +1317,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the ifunc information. for (const GlobalIFunc &I : M.ifuncs()) { // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver - // val#, linkage, visibility] + // val#, linkage, visibility, DSO_Local] Vals.push_back(addToStrtab(I.getName())); Vals.push_back(I.getName().size()); Vals.push_back(VE.getTypeID(I.getValueType())); @@ -1310,6 +1325,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(VE.getValueID(I.getResolver())); Vals.push_back(getEncodedLinkage(I)); Vals.push_back(getEncodedVisibility(I)); + Vals.push_back(I.isDSOLocal()); Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals); Vals.clear(); } @@ -1330,19 +1346,19 @@ static uint64_t getOptimizationFlags(const Value *V) { Flags |= 1 << bitc::PEO_EXACT; } else if (const auto *FPMO = dyn_cast<FPMathOperator>(V)) { if (FPMO->hasAllowReassoc()) - Flags |= FastMathFlags::AllowReassoc; + Flags |= bitc::AllowReassoc; if (FPMO->hasNoNaNs()) - Flags |= FastMathFlags::NoNaNs; + Flags |= bitc::NoNaNs; if (FPMO->hasNoInfs()) - Flags |= FastMathFlags::NoInfs; + Flags |= bitc::NoInfs; if (FPMO->hasNoSignedZeros()) - Flags |= FastMathFlags::NoSignedZeros; + Flags |= bitc::NoSignedZeros; if (FPMO->hasAllowReciprocal()) - Flags |= FastMathFlags::AllowReciprocal; + Flags |= bitc::AllowReciprocal; if (FPMO->hasAllowContract()) - Flags |= FastMathFlags::AllowContract; + Flags |= bitc::AllowContract; if (FPMO->hasApproxFunc()) - Flags |= FastMathFlags::ApproxFunc; + Flags |= bitc::ApproxFunc; } return Flags; @@ -1441,8 +1457,9 @@ static uint64_t rotateSign(int64_t I) { void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); - Record.push_back(N->getCount()); + const uint64_t Version = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | Version); + Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode())); Record.push_back(rotateSign(N->getLowerBound())); Stream.EmitRecord(bitc::METADATA_SUBRANGE, Record, Abbrev); @@ -1452,7 +1469,7 @@ void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N, void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + Record.push_back((N->isUnsigned() << 1) | N->isDistinct()); Record.push_back(rotateSign(N->getValue())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1521,6 +1538,7 @@ void ModuleBitcodeWriter::writeDICompositeType( Record.push_back(VE.getMetadataOrNullID(N->getVTableHolder())); Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier())); + Record.push_back(VE.getMetadataOrNullID(N->getDiscriminator())); Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev); Record.clear(); @@ -1545,8 +1563,18 @@ void ModuleBitcodeWriter::writeDIFile(const DIFile *N, Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawFilename())); Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory())); - Record.push_back(N->getChecksumKind()); - Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum())); + if (N->getRawChecksum()) { + Record.push_back(N->getRawChecksum()->Kind); + Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum()->Value)); + } else { + // Maintain backwards compatibility with the old internal representation of + // CSK_None in ChecksumKind by writing nulls here when Checksum is None. + Record.push_back(0); + Record.push_back(VE.getMetadataOrNullID(nullptr)); + } + auto Source = N->getRawSource(); + if (Source) + Record.push_back(VE.getMetadataOrNullID(*Source)); Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev); Record.clear(); @@ -1602,7 +1630,7 @@ void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N, Record.push_back(VE.getMetadataOrNullID(N->getRawUnit())); Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getDeclaration())); - Record.push_back(VE.getMetadataOrNullID(N->getVariables().get())); + Record.push_back(VE.getMetadataOrNullID(N->getRetainedNodes().get())); Record.push_back(N->getThisAdjustment()); Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get())); @@ -1759,6 +1787,19 @@ void ModuleBitcodeWriter::writeDILocalVariable( Record.clear(); } +void ModuleBitcodeWriter::writeDILabel( + const DILabel *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + Record.push_back((uint64_t)N->isDistinct()); + Record.push_back(VE.getMetadataOrNullID(N->getScope())); + Record.push_back(VE.getMetadataOrNullID(N->getRawName())); + Record.push_back(VE.getMetadataOrNullID(N->getFile())); + Record.push_back(N->getLine()); + + Stream.EmitRecord(bitc::METADATA_LABEL, Record, Abbrev); + Record.clear(); +} + void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { @@ -3183,7 +3224,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_BINOP_FLAGS_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); @@ -3312,10 +3353,14 @@ void IndexBitcodeWriter::writeModStrings() { /// Write the function type metadata related records that need to appear before /// a function summary entry (whether per-module or combined). -static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, - FunctionSummary *FS) { - if (!FS->type_tests().empty()) +static void writeFunctionTypeMetadataRecords( + BitstreamWriter &Stream, FunctionSummary *FS, + std::set<GlobalValue::GUID> &ReferencedTypeIds) { + if (!FS->type_tests().empty()) { Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + for (auto &TT : FS->type_tests()) + ReferencedTypeIds.insert(TT); + } SmallVector<uint64_t, 64> Record; @@ -3327,6 +3372,7 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, for (auto &VF : VFs) { Record.push_back(VF.GUID); Record.push_back(VF.Offset); + ReferencedTypeIds.insert(VF.GUID); } Stream.EmitRecord(Ty, Record); }; @@ -3341,6 +3387,7 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, for (auto &VC : VCs) { Record.clear(); Record.push_back(VC.VFunc.GUID); + ReferencedTypeIds.insert(VC.VFunc.GUID); Record.push_back(VC.VFunc.Offset); Record.insert(Record.end(), VC.Args.begin(), VC.Args.end()); Stream.EmitRecord(Ty, Record); @@ -3353,6 +3400,51 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, FS->type_checked_load_const_vcalls()); } +static void writeWholeProgramDevirtResolutionByArg( + SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args, + const WholeProgramDevirtResolution::ByArg &ByArg) { + NameVals.push_back(args.size()); + NameVals.insert(NameVals.end(), args.begin(), args.end()); + + NameVals.push_back(ByArg.TheKind); + NameVals.push_back(ByArg.Info); + NameVals.push_back(ByArg.Byte); + NameVals.push_back(ByArg.Bit); +} + +static void writeWholeProgramDevirtResolution( + SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder, + uint64_t Id, const WholeProgramDevirtResolution &Wpd) { + NameVals.push_back(Id); + + NameVals.push_back(Wpd.TheKind); + NameVals.push_back(StrtabBuilder.add(Wpd.SingleImplName)); + NameVals.push_back(Wpd.SingleImplName.size()); + + NameVals.push_back(Wpd.ResByArg.size()); + for (auto &A : Wpd.ResByArg) + writeWholeProgramDevirtResolutionByArg(NameVals, A.first, A.second); +} + +static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals, + StringTableBuilder &StrtabBuilder, + const std::string &Id, + const TypeIdSummary &Summary) { + NameVals.push_back(StrtabBuilder.add(Id)); + NameVals.push_back(Id.size()); + + NameVals.push_back(Summary.TTRes.TheKind); + NameVals.push_back(Summary.TTRes.SizeM1BitWidth); + NameVals.push_back(Summary.TTRes.AlignLog2); + NameVals.push_back(Summary.TTRes.SizeM1); + NameVals.push_back(Summary.TTRes.BitMask); + NameVals.push_back(Summary.TTRes.InlineBits); + + for (auto &W : Summary.WPDRes) + writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, W.first, + W.second); +} + // Helper to emit a single function summary record. void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, @@ -3361,7 +3453,8 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( NameVals.push_back(ValueID); FunctionSummary *FS = cast<FunctionSummary>(Summary); - writeFunctionTypeMetadataRecords(Stream, FS); + std::set<GlobalValue::GUID> ReferencedTypeIds; + writeFunctionTypeMetadataRecords(Stream, FS, ReferencedTypeIds); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); @@ -3371,16 +3464,21 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( for (auto &RI : FS->refs()) NameVals.push_back(VE.getValueID(RI.getValue())); - bool HasProfileData = F.hasProfileData(); + bool HasProfileData = + F.hasProfileData() || ForceSummaryEdgesCold != FunctionSummary::FSHT_None; for (auto &ECI : FS->calls()) { NameVals.push_back(getValueId(ECI.first)); if (HasProfileData) NameVals.push_back(static_cast<uint8_t>(ECI.second.Hotness)); + else if (WriteRelBFToSummary) + NameVals.push_back(ECI.second.RelBlockFreq); } unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); unsigned Code = - (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE); + (HasProfileData ? bitc::FS_PERMODULE_PROFILE + : (WriteRelBFToSummary ? bitc::FS_PERMODULE_RELBF + : bitc::FS_PERMODULE)); // Emit the finished record. Stream.EmitRecord(Code, NameVals, FSAbbrev); @@ -3392,7 +3490,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( void ModuleBitcodeWriterBase::writeModuleLevelReferences( const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev) { - auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName())); + auto VI = Index->getValueInfo(V.getGUID()); if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might however // have a summary if the def was in module level asm). @@ -3409,7 +3507,7 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences( NameVals.push_back(VE.getValueID(RI.getValue())); // Sort the refs for determinism output, the vector returned by FS->refs() has // been initialized from a DenseSet. - std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); + llvm::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, FSModRefsAbbrev); @@ -3446,31 +3544,34 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { ArrayRef<uint64_t>{GVI.second, GVI.first}); } - // Abbrev for FS_PERMODULE. + // Abbrev for FS_PERMODULE_PROFILE. auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid) + // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - // Abbrev for FS_PERMODULE_PROFILE. + // Abbrev for FS_PERMODULE or FS_PERMODULE_RELBF. Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); + if (WriteRelBFToSummary) + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF)); + else + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, hotness) + // numrefs x valueid, n x (valueid [, rel_block_freq]) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. Abbv = std::make_shared<BitCodeAbbrev>(); @@ -3498,7 +3599,7 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - ValueInfo VI = Index->getValueInfo(GlobalValue::getGUID(F.getName())); + ValueInfo VI = Index->getValueInfo(F.getGUID()); if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might // however have a summary if the def was in module level asm). @@ -3539,6 +3640,14 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + // Write the index flags. + uint64_t Flags = 0; + if (Index.withGlobalValueDeadStripping()) + Flags |= 0x1; + if (Index.skipModuleByDistributedBackend()) + Flags |= 0x2; + Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags}); + for (const auto &GVI : valueIds()) { Stream.EmitRecord(bitc::FS_VALUE_GUID, ArrayRef<uint64_t>{GVI.second, GVI.first}); @@ -3600,6 +3709,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { SmallVector<uint64_t, 64> NameVals; + // Set that will be populated during call to writeFunctionTypeMetadataRecords + // with the type ids referenced by this index file. + std::set<GlobalValue::GUID> ReferencedTypeIds; + // For local linkage, we also emit the original name separately // immediately after the record. auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) { @@ -3651,7 +3764,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { } auto *FS = cast<FunctionSummary>(S); - writeFunctionTypeMetadataRecords(Stream, FS); + writeFunctionTypeMetadataRecords(Stream, FS, ReferencedTypeIds); NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); @@ -3673,7 +3786,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { bool HasProfileData = false; for (auto &EI : FS->calls()) { - HasProfileData |= EI.second.Hotness != CalleeInfo::HotnessType::Unknown; + HasProfileData |= + EI.second.getHotness() != CalleeInfo::HotnessType::Unknown; if (HasProfileData) break; } @@ -3757,6 +3871,17 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.clear(); } + if (!Index.typeIds().empty()) { + for (auto &S : Index.typeIds()) { + // Skip if not referenced in any GV summary within this index file. + if (!ReferencedTypeIds.count(GlobalValue::getGUID(S.first))) + continue; + writeTypeIdSummaryRecord(NameVals, StrtabBuilder, S.first, S.second); + Stream.EmitRecord(bitc::FS_TYPE_ID, NameVals); + NameVals.clear(); + } + } + Stream.ExitBlock(); } @@ -4012,7 +4137,7 @@ void BitcodeWriter::copyStrtab(StringRef Strtab) { WroteStrtab = true; } -void BitcodeWriter::writeModule(const Module *M, +void BitcodeWriter::writeModule(const Module &M, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, bool GenerateHash, ModuleHash *ModHash) { @@ -4022,8 +4147,8 @@ void BitcodeWriter::writeModule(const Module *M, // Modules in case it needs to materialize metadata. But the bitcode writer // requires that the module is materialized, so we can cast to non-const here, // after checking that it is in fact materialized. - assert(M->isMaterialized()); - Mods.push_back(const_cast<Module *>(M)); + assert(M.isMaterialized()); + Mods.push_back(const_cast<Module *>(&M)); ModuleBitcodeWriter ModuleWriter(M, Buffer, StrtabBuilder, *Stream, ShouldPreserveUseListOrder, Index, @@ -4039,9 +4164,8 @@ void BitcodeWriter::writeIndex( IndexWriter.write(); } -/// WriteBitcodeToFile - Write the specified module to the specified output -/// stream. -void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, +/// Write the specified module to the specified output stream. +void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, bool GenerateHash, ModuleHash *ModHash) { @@ -4050,7 +4174,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, // If this is darwin or another generic macho target, reserve space for the // header. - Triple TT(M->getTargetTriple()); + Triple TT(M.getTargetTriple()); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); @@ -4107,7 +4231,7 @@ class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase { const ModuleHash *ModHash; public: - ThinLinkBitcodeWriter(const Module *M, StringTableBuilder &StrtabBuilder, + ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder, BitstreamWriter &Stream, const ModuleSummaryIndex &Index, const ModuleHash &ModHash) @@ -4225,7 +4349,7 @@ void ThinLinkBitcodeWriter::write() { Stream.ExitBlock(); } -void BitcodeWriter::writeThinLinkBitcode(const Module *M, +void BitcodeWriter::writeThinLinkBitcode(const Module &M, const ModuleSummaryIndex &Index, const ModuleHash &ModHash) { assert(!WroteStrtab); @@ -4234,8 +4358,8 @@ void BitcodeWriter::writeThinLinkBitcode(const Module *M, // Modules in case it needs to materialize metadata. But the bitcode writer // requires that the module is materialized, so we can cast to non-const here, // after checking that it is in fact materialized. - assert(M->isMaterialized()); - Mods.push_back(const_cast<Module *>(M)); + assert(M.isMaterialized()); + Mods.push_back(const_cast<Module *>(&M)); ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index, ModHash); @@ -4245,7 +4369,7 @@ void BitcodeWriter::writeThinLinkBitcode(const Module *M, // Write the specified thin link bitcode file to the given raw output stream, // where it will be written in a new bitcode block. This is used when // writing the per-module index file for ThinLTO. -void llvm::WriteThinLinkBitcodeToFile(const Module *M, raw_ostream &Out, +void llvm::WriteThinLinkBitcodeToFile(const Module &M, raw_ostream &Out, const ModuleSummaryIndex &Index, const ModuleHash &ModHash) { SmallVector<char, 0> Buffer; diff --git a/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 80cab762a68c..41212e575f8e 100644 --- a/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -23,7 +23,7 @@ PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { const ModuleSummaryIndex *Index = EmitSummaryIndex ? &(AM.getResult<ModuleSummaryIndexAnalysis>(M)) : nullptr; - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash); + WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash); return PreservedAnalyses::all(); } @@ -55,7 +55,7 @@ namespace { EmitSummaryIndex ? &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex()) : nullptr; - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, + WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash); return false; } @@ -80,3 +80,7 @@ ModulePass *llvm::createBitcodeWriterPass(raw_ostream &Str, return new WriteBitcodePass(Str, ShouldPreserveUseListOrder, EmitSummaryIndex, EmitModuleHash); } + +bool llvm::isBitcodeWriterPass(Pass *P) { + return P->getPassID() == (llvm::AnalysisID)&WriteBitcodePass::ID; +} diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp index d99befcdaeae..d473741e8ceb 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -14,6 +14,7 @@ #include "ValueEnumerator.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Config/llvm-config.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" @@ -183,7 +184,7 @@ static void predictValueUseListOrderImpl(const Value *V, const Function *F, return; bool IsGlobalValue = OM.isGlobalValue(ID); - std::sort(List.begin(), List.end(), [&](const Entry &L, const Entry &R) { + llvm::sort(List.begin(), List.end(), [&](const Entry &L, const Entry &R) { const Use *LU = L.first; const Use *RU = R.first; if (LU == RU) @@ -488,7 +489,7 @@ void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map, V->print(errs()); errs() << '\n'; - OS << " Uses(" << std::distance(V->use_begin(),V->use_end()) << "):"; + OS << " Uses(" << V->getNumUses() << "):"; for (const Use &U : V->uses()) { if (&U != &*V->use_begin()) OS << ","; @@ -744,7 +745,7 @@ void ValueEnumerator::organizeMetadata() { // and then sort by the original/current ID. Since the IDs are guaranteed to // be unique, the result of std::sort will be deterministic. There's no need // for std::stable_sort. - std::sort(Order.begin(), Order.end(), [this](MDIndex LHS, MDIndex RHS) { + llvm::sort(Order.begin(), Order.end(), [this](MDIndex LHS, MDIndex RHS) { return std::make_tuple(LHS.F, getMetadataTypeOrder(LHS.get(MDs)), LHS.ID) < std::make_tuple(RHS.F, getMetadataTypeOrder(RHS.get(MDs)), RHS.ID); }); |