diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2017-12-18 20:10:56 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2017-12-18 20:10:56 +0000 |
commit | 044eb2f6afba375a914ac9d8024f8f5142bb912e (patch) | |
tree | 1475247dc9f9fe5be155ebd4c9069c75aadf8c20 /lib/Bitcode | |
parent | eb70dddbd77e120e5d490bd8fbe7ff3f8fa81c6b (diff) |
Notes
Diffstat (limited to 'lib/Bitcode')
-rw-r--r-- | lib/Bitcode/Reader/BitReader.cpp | 1 | ||||
-rw-r--r-- | lib/Bitcode/Reader/BitcodeReader.cpp | 228 | ||||
-rw-r--r-- | lib/Bitcode/Reader/MetadataLoader.cpp | 31 | ||||
-rw-r--r-- | lib/Bitcode/Reader/ValueList.cpp | 29 | ||||
-rw-r--r-- | lib/Bitcode/Reader/ValueList.h | 20 | ||||
-rw-r--r-- | lib/Bitcode/Writer/BitcodeWriter.cpp | 461 | ||||
-rw-r--r-- | lib/Bitcode/Writer/ValueEnumerator.cpp | 52 | ||||
-rw-r--r-- | lib/Bitcode/Writer/ValueEnumerator.h | 59 |
8 files changed, 635 insertions, 246 deletions
diff --git a/lib/Bitcode/Reader/BitReader.cpp b/lib/Bitcode/Reader/BitReader.cpp index f64785b3ad92..3ec45956b3e5 100644 --- a/lib/Bitcode/Reader/BitReader.cpp +++ b/lib/Bitcode/Reader/BitReader.cpp @@ -10,7 +10,6 @@ #include "llvm-c/BitReader.h" #include "llvm-c/Core.h" #include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/Support/MemoryBuffer.h" diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 048e3672f471..95291a1caf9a 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -10,12 +10,11 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "MetadataLoader.h" #include "ValueList.h" - #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" @@ -33,12 +32,11 @@ #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/DiagnosticInfo.h" -#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Function.h" #include "llvm/IR/GVMaterializer.h" #include "llvm/IR/GlobalAlias.h" @@ -54,13 +52,12 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSummaryIndex.h" -#include "llvm/IR/OperandTraits.h" #include "llvm/IR/Operator.h" -#include "llvm/IR/TrackingMDRef.h" #include "llvm/IR/Type.h" -#include "llvm/IR/ValueHandle.h" +#include "llvm/IR/Value.h" #include "llvm/IR/Verifier.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Casting.h" @@ -69,7 +66,9 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> @@ -77,9 +76,9 @@ #include <cstddef> #include <cstdint> #include <deque> -#include <limits> #include <map> #include <memory> +#include <set> #include <string> #include <system_error> #include <tuple> @@ -99,13 +98,15 @@ enum { SWITCH_INST_MAGIC = 0x4B5 // May 2012 => 1205 => Hex }; -Error error(const Twine &Message) { +} // end anonymous namespace + +static Error error(const Twine &Message) { return make_error<StringError>( Message, make_error_code(BitcodeError::CorruptedBitcode)); } /// Helper to read the header common to all bitcode files. -bool hasValidBitcodeHeader(BitstreamCursor &Stream) { +static bool hasValidBitcodeHeader(BitstreamCursor &Stream) { // Sniff for the signature. if (!Stream.canSkipToPos(4) || Stream.Read(8) != 'B' || @@ -118,7 +119,7 @@ bool hasValidBitcodeHeader(BitstreamCursor &Stream) { return true; } -Expected<BitstreamCursor> initStream(MemoryBufferRef Buffer) { +static Expected<BitstreamCursor> initStream(MemoryBufferRef Buffer) { const unsigned char *BufPtr = (const unsigned char *)Buffer.getBufferStart(); const unsigned char *BufEnd = BufPtr + Buffer.getBufferSize(); @@ -151,7 +152,7 @@ static bool convertToString(ArrayRef<uint64_t> Record, unsigned Idx, } // Strip all the TBAA attachment for the module. -void stripTBAA(Module *M) { +static void stripTBAA(Module *M) { for (auto &F : *M) { if (F.isMaterializable()) continue; @@ -162,7 +163,7 @@ void stripTBAA(Module *M) { /// Read the "IDENTIFICATION_BLOCK_ID" block, do some basic enforcement on the /// "epoch" encoded in the bitcode, and return the producer name if any. -Expected<std::string> readIdentificationBlock(BitstreamCursor &Stream) { +static Expected<std::string> readIdentificationBlock(BitstreamCursor &Stream) { if (Stream.EnterSubBlock(bitc::IDENTIFICATION_BLOCK_ID)) return error("Invalid record"); @@ -206,7 +207,7 @@ Expected<std::string> readIdentificationBlock(BitstreamCursor &Stream) { } } -Expected<std::string> readIdentificationCode(BitstreamCursor &Stream) { +static Expected<std::string> readIdentificationCode(BitstreamCursor &Stream) { // We expect a number of well-defined blocks, though we don't necessarily // need to understand them all. while (true) { @@ -234,7 +235,7 @@ Expected<std::string> readIdentificationCode(BitstreamCursor &Stream) { } } -Expected<bool> hasObjCCategoryInModule(BitstreamCursor &Stream) { +static Expected<bool> hasObjCCategoryInModule(BitstreamCursor &Stream) { if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) return error("Invalid record"); @@ -275,7 +276,7 @@ Expected<bool> hasObjCCategoryInModule(BitstreamCursor &Stream) { llvm_unreachable("Exit infinite loop"); } -Expected<bool> hasObjCCategory(BitstreamCursor &Stream) { +static Expected<bool> hasObjCCategory(BitstreamCursor &Stream) { // We expect a number of well-defined blocks, though we don't necessarily // need to understand them all. while (true) { @@ -303,7 +304,7 @@ Expected<bool> hasObjCCategory(BitstreamCursor &Stream) { } } -Expected<std::string> readModuleTriple(BitstreamCursor &Stream) { +static Expected<std::string> readModuleTriple(BitstreamCursor &Stream) { if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) return error("Invalid record"); @@ -342,7 +343,7 @@ Expected<std::string> readModuleTriple(BitstreamCursor &Stream) { llvm_unreachable("Exit infinite loop"); } -Expected<std::string> readTriple(BitstreamCursor &Stream) { +static Expected<std::string> readTriple(BitstreamCursor &Stream) { // We expect a number of well-defined blocks, though we don't necessarily // need to understand them all. while (true) { @@ -370,6 +371,8 @@ Expected<std::string> readTriple(BitstreamCursor &Stream) { } } +namespace { + class BitcodeReaderBase { protected: BitcodeReaderBase(BitstreamCursor Stream, StringRef Strtab) @@ -401,6 +404,8 @@ protected: Error error(const Twine &Message); }; +} // end anonymous namespace + Error BitcodeReaderBase::error(const Twine &Message) { std::string FullMsg = Message.str(); if (!ProducerIdentification.empty()) @@ -411,7 +416,7 @@ Error BitcodeReaderBase::error(const Twine &Message) { Expected<unsigned> BitcodeReaderBase::parseVersionRecord(ArrayRef<uint64_t> Record) { - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); unsigned ModuleVersion = Record[0]; if (ModuleVersion > 2) @@ -430,6 +435,8 @@ BitcodeReaderBase::readNameFromStrtab(ArrayRef<uint64_t> Record) { return {StringRef(Strtab.data() + Record[0], Record[1]), Record.slice(2)}; } +namespace { + class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { LLVMContext &Context; Module *TheModule = nullptr; @@ -449,11 +456,11 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { std::vector<Comdat *> ComdatList; SmallVector<Instruction *, 64> InstructionList; - std::vector<std::pair<GlobalVariable*, unsigned> > GlobalInits; - std::vector<std::pair<GlobalIndirectSymbol*, unsigned> > IndirectSymbolInits; - std::vector<std::pair<Function*, unsigned> > FunctionPrefixes; - std::vector<std::pair<Function*, unsigned> > FunctionPrologues; - std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFns; + std::vector<std::pair<GlobalVariable *, unsigned>> GlobalInits; + std::vector<std::pair<GlobalIndirectSymbol *, unsigned>> IndirectSymbolInits; + std::vector<std::pair<Function *, unsigned>> FunctionPrefixes; + std::vector<std::pair<Function *, unsigned>> FunctionPrologues; + std::vector<std::pair<Function *, unsigned>> FunctionPersonalityFns; /// The set of attributes by index. Index zero in the file is for null, and /// is thus not represented here. As such all indices are off by one. @@ -472,7 +479,7 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { // When intrinsic functions are encountered which require upgrading they are // stored here with their replacement function. - typedef DenseMap<Function*, Function*> UpdatedIntrinsicMap; + using UpdatedIntrinsicMap = DenseMap<Function *, Function *>; UpdatedIntrinsicMap UpgradedIntrinsics; // Intrinsics which were remangled because of types rename UpdatedIntrinsicMap RemangledIntrinsics; @@ -860,6 +867,15 @@ static GlobalValue::LinkageTypes getDecodedLinkage(unsigned Val) { } } +static FunctionSummary::FFlags getDecodedFFlags(uint64_t RawFlags) { + FunctionSummary::FFlags Flags; + Flags.ReadNone = RawFlags & 0x1; + Flags.ReadOnly = (RawFlags >> 1) & 0x1; + Flags.NoRecurse = (RawFlags >> 2) & 0x1; + Flags.ReturnDoesNotAlias = (RawFlags >> 3) & 0x1; + return Flags; +} + /// Decode the flags for GlobalValue in the summary. static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, uint64_t Version) { @@ -873,7 +889,9 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, // to work correctly on earlier versions, we must conservatively treat all // values as live. bool Live = (RawFlags & 0x2) || Version < 3; - return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live); + bool Local = (RawFlags & 0x4); + + return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live, Local); } static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { @@ -895,6 +913,14 @@ getDecodedDLLStorageClass(unsigned Val) { } } +static bool getDecodedDSOLocal(unsigned Val) { + switch(Val) { + default: // Map unknown values to preemptable. + case 0: return false; + case 1: return true; + } +} + static GlobalVariable::ThreadLocalMode getDecodedThreadLocalMode(unsigned Val) { switch (Val) { case 0: return GlobalVariable::NotThreadLocal; @@ -1020,8 +1046,8 @@ static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) { static FastMathFlags getDecodedFastMathFlags(unsigned Val) { FastMathFlags FMF; - if (0 != (Val & FastMathFlags::UnsafeAlgebra)) - FMF.setUnsafeAlgebra(); + if (0 != (Val & FastMathFlags::AllowReassoc)) + FMF.setAllowReassoc(); if (0 != (Val & FastMathFlags::NoNaNs)) FMF.setNoNaNs(); if (0 != (Val & FastMathFlags::NoInfs)) @@ -1032,6 +1058,8 @@ static FastMathFlags getDecodedFastMathFlags(unsigned Val) { FMF.setAllowReciprocal(); if (0 != (Val & FastMathFlags::AllowContract)) FMF.setAllowContract(true); + if (0 != (Val & FastMathFlags::ApproxFunc)) + FMF.setApproxFunc(); return FMF; } @@ -1042,7 +1070,6 @@ static void upgradeDLLImportExportLinkage(GlobalValue *GV, unsigned Val) { } } - Type *BitcodeReader::getTypeByID(unsigned ID) { // The type table size is always specified correctly. if (ID >= TypeList.size()) @@ -1128,6 +1155,8 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { case Attribute::SwiftError: return 1ULL << 52; case Attribute::WriteOnly: return 1ULL << 53; case Attribute::Speculatable: return 1ULL << 54; + case Attribute::StrictFP: return 1ULL << 55; + case Attribute::SanitizeHWAddress: return 1ULL << 56; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1216,7 +1245,7 @@ Error BitcodeReader::parseAttributeBlock() { switch (Stream.readRecord(Entry.ID, Record)) { default: // Default behavior: ignore. break; - case bitc::PARAMATTR_CODE_ENTRY_OLD: { // ENTRY: [paramidx0, attr0, ...] + case bitc::PARAMATTR_CODE_ENTRY_OLD: // ENTRY: [paramidx0, attr0, ...] // FIXME: Remove in 4.0. if (Record.size() & 1) return error("Invalid record"); @@ -1230,8 +1259,7 @@ Error BitcodeReader::parseAttributeBlock() { MAttributes.push_back(AttributeList::get(Context, Attrs)); Attrs.clear(); break; - } - case bitc::PARAMATTR_CODE_ENTRY: { // ENTRY: [attrgrp0, attrgrp1, ...] + case bitc::PARAMATTR_CODE_ENTRY: // ENTRY: [attrgrp0, attrgrp1, ...] for (unsigned i = 0, e = Record.size(); i != e; ++i) Attrs.push_back(MAttributeGroups[Record[i]]); @@ -1239,7 +1267,6 @@ Error BitcodeReader::parseAttributeBlock() { Attrs.clear(); break; } - } } } @@ -1336,10 +1363,14 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::StackProtectStrong; case bitc::ATTR_KIND_SAFESTACK: return Attribute::SafeStack; + case bitc::ATTR_KIND_STRICT_FP: + return Attribute::StrictFP; case bitc::ATTR_KIND_STRUCT_RET: return Attribute::StructRet; case bitc::ATTR_KIND_SANITIZE_ADDRESS: return Attribute::SanitizeAddress; + case bitc::ATTR_KIND_SANITIZE_HWADDRESS: + return Attribute::SanitizeHWAddress; case bitc::ATTR_KIND_SANITIZE_THREAD: return Attribute::SanitizeThread; case bitc::ATTR_KIND_SANITIZE_MEMORY: @@ -1797,10 +1828,10 @@ Expected<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record, auto *GO = dyn_cast<GlobalObject>(V); if (GO) { if (GO->getComdat() == reinterpret_cast<Comdat *>(1)) { - if (TT.isOSBinFormatMachO()) - GO->setComdat(nullptr); - else + if (TT.supportsCOMDAT()) GO->setComdat(TheModule->getOrInsertComdat(V->getName())); + else + GO->setComdat(nullptr); } } return V; @@ -1988,12 +2019,12 @@ uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) { /// Resolve all of the initializers for global values and aliases that we can. Error BitcodeReader::resolveGlobalAndIndirectSymbolInits() { - std::vector<std::pair<GlobalVariable*, unsigned> > GlobalInitWorklist; - std::vector<std::pair<GlobalIndirectSymbol*, unsigned> > + std::vector<std::pair<GlobalVariable *, unsigned>> GlobalInitWorklist; + std::vector<std::pair<GlobalIndirectSymbol *, unsigned>> IndirectSymbolInitWorklist; - std::vector<std::pair<Function*, unsigned> > FunctionPrefixWorklist; - std::vector<std::pair<Function*, unsigned> > FunctionPrologueWorklist; - std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFnWorklist; + std::vector<std::pair<Function *, unsigned>> FunctionPrefixWorklist; + std::vector<std::pair<Function *, unsigned>> FunctionPrologueWorklist; + std::vector<std::pair<Function *, unsigned>> FunctionPersonalityFnWorklist; GlobalInitWorklist.swap(GlobalInits); IndirectSymbolInitWorklist.swap(IndirectSymbolInits); @@ -2707,8 +2738,8 @@ Error BitcodeReader::globalCleanup() { // Force deallocation of memory for these vectors to favor the client that // want lazy deserialization. - std::vector<std::pair<GlobalVariable*, unsigned> >().swap(GlobalInits); - std::vector<std::pair<GlobalIndirectSymbol*, unsigned> >().swap( + std::vector<std::pair<GlobalVariable *, unsigned>>().swap(GlobalInits); + std::vector<std::pair<GlobalIndirectSymbol *, unsigned>>().swap( IndirectSymbolInits); return Error::success(); } @@ -2765,7 +2796,7 @@ Error BitcodeReader::parseComdatRecord(ArrayRef<uint64_t> Record) { StringRef Name; std::tie(Name, Record) = readNameFromStrtab(Record); - if (Record.size() < 1) + if (Record.empty()) return error("Invalid record"); Comdat::SelectionKind SK = getDecodedComdatSelectionKind(Record[0]); std::string OldFormatName; @@ -2787,7 +2818,7 @@ Error BitcodeReader::parseComdatRecord(ArrayRef<uint64_t> Record) { Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) { // v1: [pointer type, isconst, initid, linkage, alignment, section, // visibility, threadlocal, unnamed_addr, externally_initialized, - // dllstorageclass, comdat, attributes] (name in VST) + // dllstorageclass, comdat, attributes, preemption specifier] (name in VST) // v2: [strtab_offset, strtab_size, v1] StringRef Name; std::tie(Name, Record) = readNameFromStrtab(Record); @@ -2872,13 +2903,18 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) { auto AS = getAttributes(Record[12]).getFnAttributes(); NewGV->setAttributes(AS); } + + if (Record.size() > 13) { + NewGV->setDSOLocal(getDecodedDSOLocal(Record[13])); + } + return Error::success(); } Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) { // v1: [type, callingconv, isproto, linkage, paramattr, alignment, section, // visibility, gc, unnamed_addr, prologuedata, dllstorageclass, comdat, - // prefixdata] (name in VST) + // prefixdata, personalityfn, preemption specifier] (name in VST) // v2: [strtab_offset, strtab_size, v1] StringRef Name; std::tie(Name, Record) = readNameFromStrtab(Record); @@ -2952,6 +2988,10 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) { if (Record.size() > 14 && Record[14] != 0) FunctionPersonalityFns.push_back(std::make_pair(Func, Record[14] - 1)); + if (Record.size() > 15) { + Func->setDSOLocal(getDecodedDSOLocal(Record[15])); + } + ValueList.push_back(Func); // If this is a function with a body, remember the prototype we are @@ -2968,9 +3008,11 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( unsigned BitCode, ArrayRef<uint64_t> Record) { // v1 ALIAS_OLD: [alias type, aliasee val#, linkage] (name in VST) // v1 ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility, - // dllstorageclass] (name in VST) + // dllstorageclass, threadlocal, unnamed_addr, + // preemption specifier] (name in VST) // v1 IFUNC: [alias type, addrspace, aliasee val#, linkage, - // visibility, dllstorageclass] (name in VST) + // visibility, dllstorageclass, threadlocal, unnamed_addr, + // preemption specifier] (name in VST) // v2: [strtab_offset, strtab_size, v1] StringRef Name; std::tie(Name, Record) = readNameFromStrtab(Record); @@ -3020,6 +3062,8 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++])); if (OpNum != Record.size()) NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++])); + if (OpNum != Record.size()) + NewGA->setDSOLocal(getDecodedDSOLocal(Record[OpNum++])); ValueList.push_back(NewGA); IndirectSymbolInits.push_back(std::make_pair(NewGA, Val)); return Error::success(); @@ -3232,28 +3276,24 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, GCTable.push_back(S); break; } - case bitc::MODULE_CODE_COMDAT: { + case bitc::MODULE_CODE_COMDAT: if (Error Err = parseComdatRecord(Record)) return Err; break; - } - case bitc::MODULE_CODE_GLOBALVAR: { + case bitc::MODULE_CODE_GLOBALVAR: if (Error Err = parseGlobalVarRecord(Record)) return Err; break; - } - case bitc::MODULE_CODE_FUNCTION: { + case bitc::MODULE_CODE_FUNCTION: if (Error Err = parseFunctionRecord(Record)) return Err; break; - } case bitc::MODULE_CODE_IFUNC: case bitc::MODULE_CODE_ALIAS: - case bitc::MODULE_CODE_ALIAS_OLD: { + case bitc::MODULE_CODE_ALIAS_OLD: if (Error Err = parseGlobalIndirectSymbolRecord(BitCode, Record)) return Err; break; - } /// MODULE_CODE_VSTOFFSET: [offset] case bitc::MODULE_CODE_VSTOFFSET: if (Record.size() < 1) @@ -3283,7 +3323,6 @@ Error BitcodeReader::parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata, return parseModule(0, ShouldLazyLoadMetadata); } - Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) { if (!isa<PointerType>(PtrType)) return error("Load/Store operand is not a pointer type"); @@ -5036,9 +5075,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { } const uint64_t Version = Record[0]; const bool IsOldProfileFormat = Version == 1; - if (Version < 1 || Version > 3) + if (Version < 1 || Version > 4) return error("Invalid summary version " + Twine(Version) + - ", 1, 2 or 3 expected"); + ", 1, 2, 3 or 4 expected"); Record.clear(); // Keep around the last seen summary to be used when we see an optional @@ -5088,9 +5127,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID); break; } - // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid, - // n x (valueid)] - // FS_PERMODULE_PROFILE: [valueid, flags, instcount, numrefs, + // FS_PERMODULE: [valueid, flags, instcount, fflags, numrefs, + // numrefs x valueid, n x (valueid)] + // FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, // n x (valueid, hotness)] case bitc::FS_PERMODULE: @@ -5098,14 +5137,21 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; unsigned InstCount = Record[2]; + uint64_t RawFunFlags = 0; unsigned NumRefs = Record[3]; + int RefListStartIndex = 4; + if (Version >= 4) { + RawFunFlags = Record[3]; + NumRefs = Record[4]; + RefListStartIndex = 5; + } + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); // The module path string ref set in the summary must be owned by the // index's module string table. Since we don't have a module path // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. - static int RefListStartIndex = 4; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); @@ -5116,8 +5162,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile); auto FS = llvm::make_unique<FunctionSummary>( - Flags, InstCount, std::move(Refs), std::move(Calls), - std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), + Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), + std::move(Calls), std::move(PendingTypeTests), + std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), std::move(PendingTypeCheckedLoadConstVCalls)); @@ -5140,8 +5187,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t RawFlags = Record[1]; unsigned AliaseeID = Record[2]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - auto AS = - llvm::make_unique<AliasSummary>(Flags, std::vector<ValueInfo>{}); + auto AS = llvm::make_unique<AliasSummary>(Flags); // The module path string ref set in the summary must be owned by the // index's module string table. Since we don't have a module path // string table section in the per-module index, we create a single @@ -5156,6 +5202,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); AS->setAliasee(AliaseeInModule); + AS->setAliaseeGUID(AliaseeGUID); auto GUID = getValueInfoFromValueId(ValueID); AS->setOriginalName(GUID.second); @@ -5176,9 +5223,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); break; } - // FS_COMBINED: [valueid, modid, flags, instcount, numrefs, + // FS_COMBINED: [valueid, modid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid)] - // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, numrefs, + // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid, hotness)] case bitc::FS_COMBINED: case bitc::FS_COMBINED_PROFILE: { @@ -5186,9 +5233,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t ModuleId = Record[1]; uint64_t RawFlags = Record[2]; unsigned InstCount = Record[3]; + uint64_t RawFunFlags = 0; unsigned NumRefs = Record[4]; + int RefListStartIndex = 5; + + if (Version >= 4) { + RawFunFlags = Record[4]; + NumRefs = Record[5]; + RefListStartIndex = 6; + } + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - static int RefListStartIndex = 5; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); @@ -5200,8 +5255,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { IsOldProfileFormat, HasProfile); ValueInfo VI = getValueInfoFromValueId(ValueID).first; auto FS = llvm::make_unique<FunctionSummary>( - Flags, InstCount, std::move(Refs), std::move(Edges), - std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), + Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), + std::move(Edges), std::move(PendingTypeTests), + std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), std::move(PendingTypeCheckedLoadConstVCalls)); @@ -5225,7 +5281,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t RawFlags = Record[2]; unsigned AliaseeValueId = Record[3]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - auto AS = llvm::make_unique<AliasSummary>(Flags, std::vector<ValueInfo>{}); + auto AS = llvm::make_unique<AliasSummary>(Flags); LastSeenSummary = AS.get(); AS->setModulePath(ModuleIdMap[ModuleId]); @@ -5233,9 +5289,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { getValueInfoFromValueId(AliaseeValueId).first.getGUID(); auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeGUID, AS->modulePath()); - if (!AliaseeInModule) - return error("Alias expects aliasee summary to be parsed"); AS->setAliasee(AliaseeInModule); + AS->setAliaseeGUID(AliaseeGUID); ValueInfo VI = getValueInfoFromValueId(ValueID).first; LastSeenGUID = VI.getGUID(); @@ -5270,34 +5325,34 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { LastSeenGUID = 0; break; } - case bitc::FS_TYPE_TESTS: { + case bitc::FS_TYPE_TESTS: assert(PendingTypeTests.empty()); PendingTypeTests.insert(PendingTypeTests.end(), Record.begin(), Record.end()); break; - } - case bitc::FS_TYPE_TEST_ASSUME_VCALLS: { + + case bitc::FS_TYPE_TEST_ASSUME_VCALLS: assert(PendingTypeTestAssumeVCalls.empty()); for (unsigned I = 0; I != Record.size(); I += 2) PendingTypeTestAssumeVCalls.push_back({Record[I], Record[I+1]}); break; - } - case bitc::FS_TYPE_CHECKED_LOAD_VCALLS: { + + case bitc::FS_TYPE_CHECKED_LOAD_VCALLS: assert(PendingTypeCheckedLoadVCalls.empty()); for (unsigned I = 0; I != Record.size(); I += 2) PendingTypeCheckedLoadVCalls.push_back({Record[I], Record[I+1]}); break; - } - case bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL: { + + case bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL: PendingTypeTestAssumeConstVCalls.push_back( {{Record[0], Record[1]}, {Record.begin() + 2, Record.end()}}); break; - } - case bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL: { + + case bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL: PendingTypeCheckedLoadConstVCalls.push_back( {{Record[0], Record[1]}, {Record.begin() + 2, Record.end()}}); break; - } + case bitc::FS_CFI_FUNCTION_DEFS: { std::set<std::string> &CfiFunctionDefs = TheIndex.cfiFunctionDefs(); for (unsigned I = 0; I != Record.size(); I += 2) @@ -5388,6 +5443,7 @@ class BitcodeErrorCategoryType : public std::error_category { const char *name() const noexcept override { return "llvm.bitcode"; } + std::string message(int IE) const override { BitcodeError E = static_cast<BitcodeError>(IE); switch (E) { @@ -5412,7 +5468,7 @@ static Expected<StringRef> readBlobInRecord(BitstreamCursor &Stream, return error("Invalid record"); StringRef Strtab; - while (1) { + while (true) { BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { case BitstreamEntry::EndBlock: diff --git a/lib/Bitcode/Reader/MetadataLoader.cpp b/lib/Bitcode/Reader/MetadataLoader.cpp index 10fbcdea784f..1b0d80d26cf5 100644 --- a/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/lib/Bitcode/Reader/MetadataLoader.cpp @@ -21,7 +21,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitstreamReader.h" @@ -30,7 +29,6 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" @@ -39,7 +37,6 @@ #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Function.h" #include "llvm/IR/GVMaterializer.h" @@ -59,7 +56,6 @@ #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/OperandTraits.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/TrackingMDRef.h" #include "llvm/IR/Type.h" #include "llvm/IR/ValueHandle.h" @@ -497,8 +493,8 @@ class MetadataLoader::MetadataLoaderImpl { for (unsigned I = 0; I < GVs->getNumOperands(); I++) if (auto *GV = dyn_cast_or_null<DIGlobalVariable>(GVs->getOperand(I))) { - auto *DGVE = - DIGlobalVariableExpression::getDistinct(Context, GV, nullptr); + auto *DGVE = DIGlobalVariableExpression::getDistinct( + Context, GV, DIExpression::get(Context, {})); GVs->replaceOperandWith(I, DGVE); } } @@ -510,8 +506,8 @@ class MetadataLoader::MetadataLoaderImpl { GV.eraseMetadata(LLVMContext::MD_dbg); for (auto *MD : MDs) if (auto *DGV = dyn_cast_or_null<DIGlobalVariable>(MD)) { - auto *DGVE = - DIGlobalVariableExpression::getDistinct(Context, DGV, nullptr); + auto *DGVE = DIGlobalVariableExpression::getDistinct( + Context, DGV, DIExpression::get(Context, {})); GV.addMetadata(LLVMContext::MD_dbg, *DGVE); } else GV.addMetadata(LLVMContext::MD_dbg, *MD); @@ -1355,7 +1351,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_COMPILE_UNIT: { - if (Record.size() < 14 || Record.size() > 18) + if (Record.size() < 14 || Record.size() > 19) return error("Invalid record"); // Ignore Record[0], which indicates whether this compile unit is @@ -1369,7 +1365,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Record.size() <= 15 ? nullptr : getMDOrNull(Record[15]), Record.size() <= 14 ? 0 : Record[14], Record.size() <= 16 ? true : Record[16], - Record.size() <= 17 ? false : Record[17]); + Record.size() <= 17 ? false : Record[17], + Record.size() <= 18 ? false : Record[18]); MetadataList.assignValue(CU, NextMetadataNo); NextMetadataNo++; @@ -1585,7 +1582,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( DIGlobalVariableExpression *DGVE = nullptr; if (Attach || Expr) - DGVE = DIGlobalVariableExpression::getDistinct(Context, DGV, Expr); + DGVE = DIGlobalVariableExpression::getDistinct( + Context, DGV, Expr ? Expr : DIExpression::get(Context, {})); if (Attach) Attach->addDebugInfo(DGVE); @@ -1648,10 +1646,13 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( return error("Invalid record"); IsDistinct = Record[0]; - MetadataList.assignValue(GET_OR_DISTINCT(DIGlobalVariableExpression, - (Context, getMDOrNull(Record[1]), - getMDOrNull(Record[2]))), - NextMetadataNo); + Metadata *Expr = getMDOrNull(Record[2]); + if (!Expr) + Expr = DIExpression::get(Context, {}); + MetadataList.assignValue( + GET_OR_DISTINCT(DIGlobalVariableExpression, + (Context, getMDOrNull(Record[1]), Expr)), + NextMetadataNo); NextMetadataNo++; break; } diff --git a/lib/Bitcode/Reader/ValueList.cpp b/lib/Bitcode/Reader/ValueList.cpp index f2a3439a87be..08bfa291098c 100644 --- a/lib/Bitcode/Reader/ValueList.cpp +++ b/lib/Bitcode/Reader/ValueList.cpp @@ -1,4 +1,4 @@ -//===----- ValueList.cpp - Internal BitcodeReader implementation ----------===// +//===- ValueList.cpp - Internal BitcodeReader implementation --------------===// // // The LLVM Compiler Infrastructure // @@ -8,27 +8,44 @@ //===----------------------------------------------------------------------===// #include "ValueList.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" -#include "llvm/IR/Instructions.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/ErrorHandling.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <limits> +#include <utility> using namespace llvm; namespace llvm { + namespace { /// \brief A class for maintaining the slot number definition /// as a placeholder for the actual definition for forward constants defs. class ConstantPlaceHolder : public ConstantExpr { - void operator=(const ConstantPlaceHolder &) = delete; - public: - // allocate space for exactly one operand - void *operator new(size_t s) { return User::operator new(s, 1); } explicit ConstantPlaceHolder(Type *Ty, LLVMContext &Context) : ConstantExpr(Ty, Instruction::UserOp1, &Op<0>(), 1) { Op<0>() = UndefValue::get(Type::getInt32Ty(Context)); } + ConstantPlaceHolder &operator=(const ConstantPlaceHolder &) = delete; + + // allocate space for exactly one operand + void *operator new(size_t s) { return User::operator new(s, 1); } + /// \brief Methods to support type inquiry through isa, cast, and dyn_cast. static bool classof(const Value *V) { return isa<ConstantExpr>(V) && diff --git a/lib/Bitcode/Reader/ValueList.h b/lib/Bitcode/Reader/ValueList.h index 72775a3cf3bc..5ad7899347ad 100644 --- a/lib/Bitcode/Reader/ValueList.h +++ b/lib/Bitcode/Reader/ValueList.h @@ -1,4 +1,4 @@ -//===-- Bitcode/Reader/ValueEnumerator.h - Number values --------*- C++ -*-===// +//===-- Bitcode/Reader/ValueList.h - Number values --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,13 +11,20 @@ // //===----------------------------------------------------------------------===// -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/ValueHandle.h" +#ifndef LLVM_LIB_BITCODE_READER_VALUELIST_H +#define LLVM_LIB_BITCODE_READER_VALUELIST_H +#include "llvm/IR/ValueHandle.h" +#include <cassert> +#include <utility> #include <vector> namespace llvm { + class Constant; +class LLVMContext; +class Type; +class Value; class BitcodeReaderValueList { std::vector<WeakTrackingVH> ValuePtrs; @@ -29,12 +36,13 @@ class BitcodeReaderValueList { /// /// The key of this vector is the placeholder constant, the value is the slot /// number that holds the resolved value. - typedef std::vector<std::pair<Constant *, unsigned>> ResolveConstantsTy; + using ResolveConstantsTy = std::vector<std::pair<Constant *, unsigned>>; ResolveConstantsTy ResolveConstants; LLVMContext &Context; public: BitcodeReaderValueList(LLVMContext &C) : Context(C) {} + ~BitcodeReaderValueList() { assert(ResolveConstants.empty() && "Constants not resolved?"); } @@ -73,4 +81,6 @@ public: void resolveConstantForwardRefs(); }; -} // namespace llvm +} // end namespace llvm + +#endif // LLVM_LIB_BITCODE_READER_VALUELIST_H diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index dcffde1742cd..fd13dbc1f1e2 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -1,4 +1,4 @@ -//===--- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ----------------===// +//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===// // // The LLVM Compiler Infrastructure // @@ -13,39 +13,81 @@ #include "llvm/Bitcode/BitcodeWriter.h" #include "ValueEnumerator.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" +#include "llvm/Bitcode/BitCodes.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" +#include "llvm/IR/GlobalObject.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InlineAsm.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/Type.h" #include "llvm/IR/UseListOrder.h" +#include "llvm/IR/Value.h" #include "llvm/IR/ValueSymbolTable.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Object/IRSymtab.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/Program.h" #include "llvm/Support/SHA1.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" -#include <cctype> +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <iterator> #include <map> -using namespace llvm; +#include <memory> +#include <string> +#include <utility> +#include <vector> -namespace { +using namespace llvm; -cl::opt<unsigned> +static cl::opt<unsigned> IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(25), cl::desc("Number of metadatas above which we emit an index " "to enable lazy-loading")); + +namespace { + /// These are manifest constants used by the bitcode writer. They do not need to /// be kept in sync with the reader, but need to be consistent within this file. enum { @@ -97,11 +139,10 @@ void BitcodeWriterBase::writeModuleVersion() { Stream.EmitRecord(bitc::MODULE_CODE_VERSION, ArrayRef<uint64_t>{2}); } -/// Class to manage the bitcode writing for a module. -class ModuleBitcodeWriter : public BitcodeWriterBase { - /// Pointer to the buffer allocated by caller for bitcode writing. - const SmallVectorImpl<char> &Buffer; - +/// Base class to manage the module bitcode writing, currently subclassed for +/// ModuleBitcodeWriter and ThinLinkBitcodeWriter. +class ModuleBitcodeWriterBase : public BitcodeWriterBase { +protected: /// The Module to write to bitcode. const Module &M; @@ -111,22 +152,6 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { /// Optional per-module index to write for ThinLTO. const ModuleSummaryIndex *Index; - /// True if a module hash record should be written. - bool GenerateHash; - - SHA1 Hasher; - - /// If non-null, when GenerateHash is true, the resulting hash is written - /// into ModHash. When GenerateHash is false, that specified value - /// is used as the hash instead of computing from the generated bitcode. - /// Can be used to produce the same module hash for a minimized bitcode - /// used just for the thin link as in the regular full bitcode that will - /// be used in the backend. - ModuleHash *ModHash; - - /// The start bit of the identification block. - uint64_t BitcodeStartBit; - /// Map that holds the correspondence between GUIDs in the summary index, /// that came from indirect call profiles, and a value id generated by this /// class to use in the VST and summary block records. @@ -140,17 +165,14 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { uint64_t VSTOffsetPlaceholder = 0; public: - /// Constructs a ModuleBitcodeWriter object for the given Module, + /// Constructs a ModuleBitcodeWriterBase object for the given Module, /// writing to the provided \p Buffer. - ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, - StringTableBuilder &StrtabBuilder, - BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - const ModuleSummaryIndex *Index, bool GenerateHash, - ModuleHash *ModHash = nullptr) - : BitcodeWriterBase(Stream, StrtabBuilder), Buffer(Buffer), M(*M), - VE(*M, ShouldPreserveUseListOrder), Index(Index), - GenerateHash(GenerateHash), ModHash(ModHash), - BitcodeStartBit(Stream.GetCurrentBitNo()) { + ModuleBitcodeWriterBase(const Module *M, StringTableBuilder &StrtabBuilder, + BitstreamWriter &Stream, + bool ShouldPreserveUseListOrder, + const ModuleSummaryIndex *Index) + : BitcodeWriterBase(Stream, StrtabBuilder), M(*M), + VE(*M, ShouldPreserveUseListOrder), Index(Index) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). @@ -172,6 +194,73 @@ public: assignValueId(CallEdge.first.getGUID()); } +protected: + void writePerModuleGlobalValueSummary(); + +private: + void writePerModuleFunctionSummaryRecord(SmallVector<uint64_t, 64> &NameVals, + GlobalValueSummary *Summary, + unsigned ValueID, + unsigned FSCallsAbbrev, + unsigned FSCallsProfileAbbrev, + const Function &F); + void writeModuleLevelReferences(const GlobalVariable &V, + SmallVector<uint64_t, 64> &NameVals, + unsigned FSModRefsAbbrev); + + void assignValueId(GlobalValue::GUID ValGUID) { + GUIDToValueIdMap[ValGUID] = ++GlobalValueId; + } + + unsigned getValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + // Expect that any GUID value had a value Id assigned by an + // earlier call to assignValueId. + assert(VMI != GUIDToValueIdMap.end() && + "GUID does not have assigned value Id"); + return VMI->second; + } + + // Helper to get the valueId for the type of value recorded in VI. + unsigned getValueId(ValueInfo VI) { + if (!VI.getValue()) + return getValueId(VI.getGUID()); + return VE.getValueID(VI.getValue()); + } + + std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } +}; + +/// Class to manage the bitcode writing for a module. +class ModuleBitcodeWriter : public ModuleBitcodeWriterBase { + /// Pointer to the buffer allocated by caller for bitcode writing. + const SmallVectorImpl<char> &Buffer; + + /// True if a module hash record should be written. + bool GenerateHash; + + /// If non-null, when GenerateHash is true, the resulting hash is written + /// into ModHash. + ModuleHash *ModHash; + + SHA1 Hasher; + + /// The start bit of the identification block. + uint64_t BitcodeStartBit; + +public: + /// Constructs a ModuleBitcodeWriter object for the given Module, + /// writing to the provided \p Buffer. + ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, + StringTableBuilder &StrtabBuilder, + BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, + const ModuleSummaryIndex *Index, bool GenerateHash, + ModuleHash *ModHash = nullptr) + : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream, + ShouldPreserveUseListOrder, Index), + Buffer(Buffer), GenerateHash(GenerateHash), ModHash(ModHash), + BitcodeStartBit(Stream.GetCurrentBitNo()) {} + /// Emit the current module to the bitstream. void write(); @@ -287,37 +376,8 @@ private: writeFunction(const Function &F, DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex); void writeBlockInfo(); - void writePerModuleFunctionSummaryRecord(SmallVector<uint64_t, 64> &NameVals, - GlobalValueSummary *Summary, - unsigned ValueID, - unsigned FSCallsAbbrev, - unsigned FSCallsProfileAbbrev, - const Function &F); - void writeModuleLevelReferences(const GlobalVariable &V, - SmallVector<uint64_t, 64> &NameVals, - unsigned FSModRefsAbbrev); - void writePerModuleGlobalValueSummary(); void writeModuleHash(size_t BlockStartPos); - void assignValueId(GlobalValue::GUID ValGUID) { - GUIDToValueIdMap[ValGUID] = ++GlobalValueId; - } - unsigned getValueId(GlobalValue::GUID ValGUID) { - const auto &VMI = GUIDToValueIdMap.find(ValGUID); - // Expect that any GUID value had a value Id assigned by an - // earlier call to assignValueId. - assert(VMI != GUIDToValueIdMap.end() && - "GUID does not have assigned value Id"); - return VMI->second; - } - // Helper to get the valueId for the type of value recorded in VI. - unsigned getValueId(ValueInfo VI) { - if (!VI.getValue()) - return getValueId(VI.getGUID()); - return VE.getValueID(VI.getValue()); - } - std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } - unsigned getEncodedSyncScopeID(SyncScope::ID SSID) { return unsigned(SSID); } @@ -353,13 +413,13 @@ public: // in writing out the call graph edges. Save the mapping from GUID // to the new global value id to use when writing those edges, which // are currently saved in the index in terms of GUID. - forEachSummary([&](GVInfo I) { + forEachSummary([&](GVInfo I, bool) { GUIDToValueIdMap[I.first] = ++GlobalValueId; }); } /// The below iterator returns the GUID and associated summary. - typedef std::pair<GlobalValue::GUID, GlobalValueSummary *> GVInfo; + using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>; /// Calls the callback for each value GUID and summary to be written to /// bitcode. This hides the details of whether they are being pulled from the @@ -368,12 +428,18 @@ public: void forEachSummary(Functor Callback) { if (ModuleToSummariesForIndex) { for (auto &M : *ModuleToSummariesForIndex) - for (auto &Summary : M.second) - Callback(Summary); + for (auto &Summary : M.second) { + Callback(Summary, false); + // Ensure aliasee is handled, e.g. for assigning a valueId, + // even if we are not importing the aliasee directly (the + // imported alias will contain a copy of aliasee). + if (auto *AS = dyn_cast<AliasSummary>(Summary.getSecond())) + Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true); + } } else { for (auto &Summaries : Index) for (auto &Summary : Summaries.second.SummaryList) - Callback({Summaries.first, Summary.get()}); + Callback({Summaries.first, Summary.get()}, false); } } @@ -413,8 +479,10 @@ private: return None; return VMI->second; } + std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } }; + } // end anonymous namespace static unsigned getEncodedCastOpcode(unsigned Opcode) { @@ -595,10 +663,14 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_STACK_PROTECT_STRONG; case Attribute::SafeStack: return bitc::ATTR_KIND_SAFESTACK; + case Attribute::StrictFP: + return bitc::ATTR_KIND_STRICT_FP; case Attribute::StructRet: return bitc::ATTR_KIND_STRUCT_RET; case Attribute::SanitizeAddress: return bitc::ATTR_KIND_SANITIZE_ADDRESS; + case Attribute::SanitizeHWAddress: + return bitc::ATTR_KIND_SANITIZE_HWADDRESS; case Attribute::SanitizeThread: return bitc::ATTR_KIND_SANITIZE_THREAD; case Attribute::SanitizeMemory: @@ -709,7 +781,6 @@ void ModuleBitcodeWriter::writeTypeTable() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned FunctionAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_ANON. @@ -718,7 +789,6 @@ void ModuleBitcodeWriter::writeTypeTable() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned StructAnonAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_NAME. @@ -734,7 +804,6 @@ void ModuleBitcodeWriter::writeTypeTable() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned StructNamedAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_ARRAY. @@ -742,7 +811,6 @@ void ModuleBitcodeWriter::writeTypeTable() { Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned ArrayAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Emit an entry count so the reader can reserve space. @@ -880,12 +948,23 @@ static unsigned getEncodedLinkage(const GlobalValue &GV) { return getEncodedLinkage(GV.getLinkage()); } +static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) { + uint64_t RawFlags = 0; + RawFlags |= Flags.ReadNone; + RawFlags |= (Flags.ReadOnly << 1); + RawFlags |= (Flags.NoRecurse << 2); + RawFlags |= (Flags.ReturnDoesNotAlias << 3); + return RawFlags; +} + // Decode the flags for GlobalValue in the summary static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { uint64_t RawFlags = 0; RawFlags |= Flags.NotEligibleToImport; // bool RawFlags |= (Flags.Live << 1); + RawFlags |= (Flags.DSOLocal << 2); + // Linkage don't need to be remapped at that time for the summary. Any future // change to the getEncodedLinkage() function will need to be taken into // account here as well. @@ -1128,7 +1207,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid, // linkage, alignment, section, visibility, threadlocal, // unnamed_addr, externally_initialized, dllstorageclass, - // comdat, attributes] + // comdat, attributes, DSO_Local] Vals.push_back(addToStrtab(GV.getName())); Vals.push_back(GV.getName().size()); Vals.push_back(VE.getTypeID(GV.getValueType())); @@ -1144,7 +1223,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { GV.isExternallyInitialized() || GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass || GV.hasComdat() || - GV.hasAttributes()) { + GV.hasAttributes() || + GV.isDSOLocal()) { Vals.push_back(getEncodedVisibility(GV)); Vals.push_back(getEncodedThreadLocalMode(GV)); Vals.push_back(getEncodedUnnamedAddr(GV)); @@ -1154,6 +1234,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { auto AL = GV.getAttributesAsList(AttributeList::FunctionIndex); Vals.push_back(VE.getAttributeListID(AL)); + + Vals.push_back(GV.isDSOLocal()); } else { AbbrevToUse = SimpleGVarAbbrev; } @@ -1167,7 +1249,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto, // linkage, paramattrs, alignment, section, visibility, gc, // unnamed_addr, prologuedata, dllstorageclass, comdat, - // prefixdata, personalityfn] + // prefixdata, personalityfn, DSO_Local] Vals.push_back(addToStrtab(F.getName())); Vals.push_back(F.getName().size()); Vals.push_back(VE.getTypeID(F.getFunctionType())); @@ -1189,6 +1271,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back( F.hasPersonalityFn() ? (VE.getValueID(F.getPersonalityFn()) + 1) : 0); + Vals.push_back(F.isDSOLocal()); unsigned AbbrevToUse = 0; Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals, AbbrevToUse); Vals.clear(); @@ -1197,7 +1280,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the alias information. for (const GlobalAlias &A : M.aliases()) { // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage, - // visibility, dllstorageclass, threadlocal, unnamed_addr] + // visibility, dllstorageclass, threadlocal, unnamed_addr, + // DSO_Local] Vals.push_back(addToStrtab(A.getName())); Vals.push_back(A.getName().size()); Vals.push_back(VE.getTypeID(A.getValueType())); @@ -1208,6 +1292,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(getEncodedDLLStorageClass(A)); Vals.push_back(getEncodedThreadLocalMode(A)); Vals.push_back(getEncodedUnnamedAddr(A)); + Vals.push_back(A.isDSOLocal()); + unsigned AbbrevToUse = 0; Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals, AbbrevToUse); Vals.clear(); @@ -1243,8 +1329,8 @@ static uint64_t getOptimizationFlags(const Value *V) { if (PEO->isExact()) Flags |= 1 << bitc::PEO_EXACT; } else if (const auto *FPMO = dyn_cast<FPMathOperator>(V)) { - if (FPMO->hasUnsafeAlgebra()) - Flags |= FastMathFlags::UnsafeAlgebra; + if (FPMO->hasAllowReassoc()) + Flags |= FastMathFlags::AllowReassoc; if (FPMO->hasNoNaNs()) Flags |= FastMathFlags::NoNaNs; if (FPMO->hasNoInfs()) @@ -1255,6 +1341,8 @@ static uint64_t getOptimizationFlags(const Value *V) { Flags |= FastMathFlags::AllowReciprocal; if (FPMO->hasAllowContract()) Flags |= FastMathFlags::AllowContract; + if (FPMO->hasApproxFunc()) + Flags |= FastMathFlags::ApproxFunc; } return Flags; @@ -1486,6 +1574,7 @@ void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, Record.push_back(VE.getMetadataOrNullID(N->getMacros().get())); Record.push_back(N->getSplitDebugInlining()); Record.push_back(N->getDebugInfoForProfiling()); + Record.push_back(N->getGnuPubnames()); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -1688,7 +1777,7 @@ void ModuleBitcodeWriter::writeDIGlobalVariableExpression( Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getVariable())); Record.push_back(VE.getMetadataOrNullID(N->getExpression())); - + Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR_EXPR, Record, Abbrev); Record.clear(); } @@ -2180,7 +2269,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, Record.push_back(p[0]); Record.push_back(p[1]); } else { - assert (0 && "Unknown FP type!"); + assert(0 && "Unknown FP type!"); } } else if (isa<ConstantDataSequential>(C) && cast<ConstantDataSequential>(C)->isString()) { @@ -3025,8 +3114,6 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } - - { // SETTYPE abbrev for CONSTANTS_BLOCK. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE)); @@ -3267,7 +3354,7 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, } // Helper to emit a single function summary record. -void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( +void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, const Function &F) { @@ -3278,6 +3365,7 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); + NameVals.push_back(getEncodedFFlags(FS->fflags())); NameVals.push_back(FS->refs().size()); for (auto &RI : FS->refs()) @@ -3301,7 +3389,7 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( // Collect the global value references in the given variable's initializer, // and emit them in a summary record. -void ModuleBitcodeWriter::writeModuleLevelReferences( +void ModuleBitcodeWriterBase::writeModuleLevelReferences( const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev) { auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName())); @@ -3331,11 +3419,11 @@ void ModuleBitcodeWriter::writeModuleLevelReferences( // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 3; +static const uint64_t INDEX_VERSION = 4; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. -void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { +void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { // By default we compile with ThinLTO if the module has a summary, but the // client can request full LTO with a module flag. bool IsThinLTO = true; @@ -3364,6 +3452,7 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3376,6 +3465,7 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3461,6 +3551,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3474,6 +3565,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3518,7 +3610,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.clear(); }; - forEachSummary([&](GVInfo I) { + forEachSummary([&](GVInfo I, bool IsAliasee) { GlobalValueSummary *S = I.second; assert(S); @@ -3526,6 +3618,12 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { assert(ValueId); SummaryToValueIdMap[S] = *ValueId; + // If this is invoked for an aliasee, we want to record the above + // mapping, but then not emit a summary entry (if the aliasee is + // to be imported, we will invoke this separately with IsAliasee=false). + if (IsAliasee) + return; + if (auto *AS = dyn_cast<AliasSummary>(S)) { // Will process aliases as a post-pass because the reader wants all // global to be loaded first. @@ -3559,6 +3657,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); + NameVals.push_back(getEncodedFFlags(FS->fflags())); // Fill in below NameVals.push_back(0); @@ -3570,7 +3669,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.push_back(*RefValueId); Count++; } - NameVals[4] = Count; + NameVals[5] = Count; bool HasProfileData = false; for (auto &EI : FS->calls()) { @@ -3594,6 +3693,20 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { CallValueId = getValueId(GUID); if (!CallValueId) continue; + // The mapping from OriginalId to GUID may return a GUID + // that corresponds to a static variable. Filter it out here. + // This can happen when + // 1) There is a call to a library function which does not have + // a CallValidId; + // 2) There is a static variable with the OriginalGUID identical + // to the GUID of the library function in 1); + // When this happens, the logic for SamplePGO kicks in and + // the static variable in 2) will be found, which needs to be + // filtered out. + auto *GVSum = Index.getGlobalValueSummary(GUID, false); + if (GVSum && + GVSum->getSummaryKind() == GlobalValueSummary::GlobalVarKind) + continue; } NameVals.push_back(*CallValueId); if (HasProfileData) @@ -3689,8 +3802,7 @@ void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { if (ModHash) // Save the written hash value. std::copy(std::begin(Vals), std::end(Vals), std::begin(*ModHash)); - } else if (ModHash) - Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef<uint32_t>(*ModHash)); + } } void ModuleBitcodeWriter::write() { @@ -3985,3 +4097,164 @@ void llvm::WriteIndexToFile( Out.write((char *)&Buffer.front(), Buffer.size()); } + +namespace { + +/// Class to manage the bitcode writing for a thin link bitcode file. +class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase { + /// ModHash is for use in ThinLTO incremental build, generated while writing + /// the module bitcode file. + const ModuleHash *ModHash; + +public: + ThinLinkBitcodeWriter(const Module *M, StringTableBuilder &StrtabBuilder, + BitstreamWriter &Stream, + const ModuleSummaryIndex &Index, + const ModuleHash &ModHash) + : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream, + /*ShouldPreserveUseListOrder=*/false, &Index), + ModHash(&ModHash) {} + + void write(); + +private: + void writeSimplifiedModuleInfo(); +}; + +} // end anonymous namespace + +// This function writes a simpilified module info for thin link bitcode file. +// It only contains the source file name along with the name(the offset and +// size in strtab) and linkage for global values. For the global value info +// entry, in order to keep linkage at offset 5, there are three zeros used +// as padding. +void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() { + SmallVector<unsigned, 64> Vals; + // Emit the module's source file name. + { + StringEncoding Bits = getStringEncoding(M.getSourceFileName()); + BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); + if (Bits == SE_Char6) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); + else if (Bits == SE_Fixed7) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); + + // MODULE_CODE_SOURCE_FILENAME: [namechar x N] + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(AbbrevOpToUse); + unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + for (const auto P : M.getSourceFileName()) + Vals.push_back((unsigned char)P); + + Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); + Vals.clear(); + } + + // Emit the global variable information. + for (const GlobalVariable &GV : M.globals()) { + // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage] + Vals.push_back(StrtabBuilder.add(GV.getName())); + Vals.push_back(GV.getName().size()); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(getEncodedLinkage(GV)); + + Stream.EmitRecord(bitc::MODULE_CODE_GLOBALVAR, Vals); + Vals.clear(); + } + + // Emit the function proto information. + for (const Function &F : M) { + // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage] + Vals.push_back(StrtabBuilder.add(F.getName())); + Vals.push_back(F.getName().size()); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(getEncodedLinkage(F)); + + Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals); + Vals.clear(); + } + + // Emit the alias information. + for (const GlobalAlias &A : M.aliases()) { + // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage] + Vals.push_back(StrtabBuilder.add(A.getName())); + Vals.push_back(A.getName().size()); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(getEncodedLinkage(A)); + + Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals); + Vals.clear(); + } + + // Emit the ifunc information. + for (const GlobalIFunc &I : M.ifuncs()) { + // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage] + Vals.push_back(StrtabBuilder.add(I.getName())); + Vals.push_back(I.getName().size()); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(0); + Vals.push_back(getEncodedLinkage(I)); + + Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals); + Vals.clear(); + } +} + +void ThinLinkBitcodeWriter::write() { + Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); + + writeModuleVersion(); + + writeSimplifiedModuleInfo(); + + writePerModuleGlobalValueSummary(); + + // Write module hash. + Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef<uint32_t>(*ModHash)); + + Stream.ExitBlock(); +} + +void BitcodeWriter::writeThinLinkBitcode(const Module *M, + const ModuleSummaryIndex &Index, + const ModuleHash &ModHash) { + assert(!WroteStrtab); + + // The Mods vector is used by irsymtab::build, which requires non-const + // Modules in case it needs to materialize metadata. But the bitcode writer + // requires that the module is materialized, so we can cast to non-const here, + // after checking that it is in fact materialized. + assert(M->isMaterialized()); + Mods.push_back(const_cast<Module *>(M)); + + ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index, + ModHash); + ThinLinkWriter.write(); +} + +// Write the specified thin link bitcode file to the given raw output stream, +// where it will be written in a new bitcode block. This is used when +// writing the per-module index file for ThinLTO. +void llvm::WriteThinLinkBitcodeToFile(const Module *M, raw_ostream &Out, + const ModuleSummaryIndex &Index, + const ModuleHash &ModHash) { + SmallVector<char, 0> Buffer; + Buffer.reserve(256 * 1024); + + BitcodeWriter Writer(Buffer); + Writer.writeThinLinkBitcode(M, Index, ModHash); + Writer.writeSymtab(); + Writer.writeStrtab(); + + Out.write((char *)&Buffer.front(), Buffer.size()); +} diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp index bb626baabd12..d99befcdaeae 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -1,4 +1,4 @@ -//===-- ValueEnumerator.cpp - Number values and types for bitcode writer --===// +//===- ValueEnumerator.cpp - Number values and types for bitcode writer ---===// // // The LLVM Compiler Infrastructure // @@ -12,47 +12,77 @@ //===----------------------------------------------------------------------===// #include "ValueEnumerator.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/IR/Constants.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" +#include "llvm/IR/GlobalObject.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Use.h" #include "llvm/IR/UseListOrder.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" #include "llvm/IR/ValueSymbolTable.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> +#include <cassert> +#include <cstddef> +#include <iterator> +#include <tuple> +#include <utility> +#include <vector> + using namespace llvm; namespace { + struct OrderMap { DenseMap<const Value *, std::pair<unsigned, bool>> IDs; - unsigned LastGlobalConstantID; - unsigned LastGlobalValueID; + unsigned LastGlobalConstantID = 0; + unsigned LastGlobalValueID = 0; - OrderMap() : LastGlobalConstantID(0), LastGlobalValueID(0) {} + OrderMap() = default; bool isGlobalConstant(unsigned ID) const { return ID <= LastGlobalConstantID; } + bool isGlobalValue(unsigned ID) const { return ID <= LastGlobalValueID && !isGlobalConstant(ID); } unsigned size() const { return IDs.size(); } std::pair<unsigned, bool> &operator[](const Value *V) { return IDs[V]; } + std::pair<unsigned, bool> lookup(const Value *V) const { return IDs.lookup(V); } + void index(const Value *V) { // Explicitly sequence get-size and insert-value operations to avoid UB. unsigned ID = IDs.size() + 1; IDs[V].first = ID; } }; -} + +} // end anonymous namespace static void orderValue(const Value *V, OrderMap &OM) { if (OM.lookup(V).first) @@ -141,7 +171,7 @@ static void predictValueUseListOrderImpl(const Value *V, const Function *F, unsigned ID, const OrderMap &OM, UseListOrderStack &Stack) { // Predict use-list order for this one. - typedef std::pair<const Use *, unsigned> Entry; + using Entry = std::pair<const Use *, unsigned>; SmallVector<Entry, 64> List; for (const Use &U : V->uses()) // Check if this user will be serialized. @@ -446,12 +476,10 @@ LLVM_DUMP_METHOD void ValueEnumerator::dump() const { void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map, const char *Name) const { - OS << "Map Name: " << Name << "\n"; OS << "Size: " << Map.size() << "\n"; for (ValueMapType::const_iterator I = Map.begin(), E = Map.end(); I != E; ++I) { - const Value *V = I->first; if (V->hasName()) OS << "Value: " << V->getName(); @@ -476,7 +504,6 @@ void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map, void ValueEnumerator::print(raw_ostream &OS, const MetadataMapType &Map, const char *Name) const { - OS << "Map Name: " << Name << "\n"; OS << "Size: " << Map.size() << "\n"; for (auto I = Map.begin(), E = Map.end(); I != E; ++I) { @@ -518,7 +545,6 @@ void ValueEnumerator::OptimizeConstants(unsigned CstStart, unsigned CstEnd) { ValueMap[Values[CstStart].first] = CstStart+1; } - /// EnumerateValueSymbolTable - Insert all of the values in the specified symbol /// table into the values table. void ValueEnumerator::EnumerateValueSymbolTable(const ValueSymbolTable &VST) { diff --git a/lib/Bitcode/Writer/ValueEnumerator.h b/lib/Bitcode/Writer/ValueEnumerator.h index e7ccc8df1e5f..011356c32601 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.h +++ b/lib/Bitcode/Writer/ValueEnumerator.h @@ -1,4 +1,4 @@ -//===-- Bitcode/Writer/ValueEnumerator.h - Number values --------*- C++ -*-===// +//===- Bitcode/Writer/ValueEnumerator.h - Number values ---------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,56 +14,57 @@ #ifndef LLVM_LIB_BITCODE_WRITER_VALUEENUMERATOR_H #define LLVM_LIB_BITCODE_WRITER_VALUEENUMERATOR_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/UniqueVector.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Type.h" #include "llvm/IR/UseListOrder.h" +#include <cassert> +#include <cstdint> +#include <utility> #include <vector> namespace llvm { -class Type; -class Value; -class Instruction; class BasicBlock; class Comdat; class Function; -class Module; -class Metadata; +class Instruction; class LocalAsMetadata; class MDNode; -class MDOperand; +class Metadata; +class Module; class NamedMDNode; -class AttributeList; -class ValueSymbolTable; -class MDSymbolTable; class raw_ostream; +class Type; +class Value; +class ValueSymbolTable; class ValueEnumerator { public: - typedef std::vector<Type*> TypeList; + using TypeList = std::vector<Type *>; // For each value, we remember its Value* and occurrence frequency. - typedef std::vector<std::pair<const Value*, unsigned> > ValueList; + using ValueList = std::vector<std::pair<const Value *, unsigned>>; /// Attribute groups as encoded in bitcode are almost AttributeSets, but they /// include the AttributeList index, so we have to track that in our map. - typedef std::pair<unsigned, AttributeSet> IndexAndAttrSet; + using IndexAndAttrSet = std::pair<unsigned, AttributeSet>; UseListOrderStack UseListOrders; private: - typedef DenseMap<Type*, unsigned> TypeMapType; + using TypeMapType = DenseMap<Type *, unsigned>; TypeMapType TypeMap; TypeList Types; - typedef DenseMap<const Value*, unsigned> ValueMapType; + using ValueMapType = DenseMap<const Value *, unsigned>; ValueMapType ValueMap; ValueList Values; - typedef UniqueVector<const Comdat *> ComdatSetType; + using ComdatSetType = UniqueVector<const Comdat *>; ComdatSetType Comdats; std::vector<const Metadata *> MDs; @@ -88,7 +89,7 @@ private: } }; - typedef DenseMap<const Metadata *, MDIndex> MetadataMapType; + using MetadataMapType = DenseMap<const Metadata *, MDIndex>; MetadataMapType MetadataMap; /// Range of metadata IDs, as a half-open range. @@ -99,18 +100,18 @@ private: /// Number of strings in the prefix of the metadata range. unsigned NumStrings = 0; - MDRange() {} + MDRange() = default; explicit MDRange(unsigned First) : First(First) {} }; SmallDenseMap<unsigned, MDRange, 1> FunctionMDInfo; bool ShouldPreserveUseListOrder; - typedef DenseMap<IndexAndAttrSet, unsigned> AttributeGroupMapType; + using AttributeGroupMapType = DenseMap<IndexAndAttrSet, unsigned>; AttributeGroupMapType AttributeGroupMap; std::vector<IndexAndAttrSet> AttributeGroups; - typedef DenseMap<AttributeList, unsigned> AttributeListMapType; + using AttributeListMapType = DenseMap<AttributeList, unsigned>; AttributeListMapType AttributeListMap; std::vector<AttributeList> AttributeLists; @@ -118,7 +119,7 @@ private: /// the "getGlobalBasicBlockID" method. mutable DenseMap<const BasicBlock*, unsigned> GlobalBasicBlockIDs; - typedef DenseMap<const Instruction*, unsigned> InstructionMapType; + using InstructionMapType = DenseMap<const Instruction *, unsigned>; InstructionMapType InstructionMap; unsigned InstructionCount; @@ -138,10 +139,10 @@ private: unsigned FirstFuncConstantID; unsigned FirstInstID; - ValueEnumerator(const ValueEnumerator &) = delete; - void operator=(const ValueEnumerator &) = delete; public: ValueEnumerator(const Module &M, bool ShouldPreserveUseListOrder); + ValueEnumerator(const ValueEnumerator &) = delete; + ValueEnumerator &operator=(const ValueEnumerator &) = delete; void dump() const; void print(raw_ostream &OS, const ValueMapType &Map, const char *Name) const; @@ -149,14 +150,17 @@ public: const char *Name) const; unsigned getValueID(const Value *V) const; + unsigned getMetadataID(const Metadata *MD) const { auto ID = getMetadataOrNullID(MD); assert(ID != 0 && "Metadata not in slotcalculator!"); return ID - 1; } + unsigned getMetadataOrNullID(const Metadata *MD) const { return MetadataMap.lookup(MD).ID; } + unsigned numMDs() const { return MDs.size(); } bool shouldPreserveUseListOrder() const { return ShouldPreserveUseListOrder; } @@ -208,10 +212,13 @@ public: } const TypeList &getTypes() const { return Types; } + const std::vector<const BasicBlock*> &getBasicBlocks() const { return BasicBlocks; } + const std::vector<AttributeList> &getAttributeLists() const { return AttributeLists; } + const std::vector<IndexAndAttrSet> &getAttributeGroups() const { return AttributeGroups; } @@ -226,8 +233,8 @@ public: /// incorporateFunction/purgeFunction - If you'd like to deal with a function, /// use these two methods to get its data into the ValueEnumerator! - /// void incorporateFunction(const Function &F); + void purgeFunction(); uint64_t computeBitsRequiredForTypeIndicies() const; @@ -292,6 +299,6 @@ private: void EnumerateNamedMetadata(const Module &M); }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_LIB_BITCODE_WRITER_VALUEENUMERATOR_H |