diff options
Diffstat (limited to 'contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp')
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 824 |
1 files changed, 370 insertions, 454 deletions
diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index ebb2022551f7..a402b4ddd462 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -28,6 +28,7 @@ #include "llvm/IR/Operator.h" #include "llvm/IR/UseListOrder.h" #include "llvm/IR/ValueSymbolTable.h" +#include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Program.h" @@ -76,26 +77,28 @@ protected: /// The stream created and owned by the client. BitstreamWriter &Stream; - /// Saves the offset of the VSTOffset record that must eventually be - /// backpatched with the offset of the actual VST. - uint64_t VSTOffsetPlaceholder = 0; - public: /// Constructs a BitcodeWriterBase object that writes to the provided /// \p Stream. BitcodeWriterBase(BitstreamWriter &Stream) : Stream(Stream) {} protected: - bool hasVSTOffsetPlaceholder() { return VSTOffsetPlaceholder != 0; } - void writeValueSymbolTableForwardDecl(); void writeBitcodeHeader(); + void writeModuleVersion(); }; +void BitcodeWriterBase::writeModuleVersion() { + // VERSION: [version#] + Stream.EmitRecord(bitc::MODULE_CODE_VERSION, ArrayRef<uint64_t>{2}); +} + /// Class to manage the bitcode writing for a module. class ModuleBitcodeWriter : public BitcodeWriterBase { /// Pointer to the buffer allocated by caller for bitcode writing. const SmallVectorImpl<char> &Buffer; + StringTableBuilder &StrtabBuilder; + /// The Module to write to bitcode. const Module &M; @@ -108,6 +111,14 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { /// True if a module hash record should be written. bool GenerateHash; + /// If non-null, when GenerateHash is true, the resulting hash is written + /// into ModHash. When GenerateHash is false, that specified value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. + ModuleHash *ModHash; + /// The start bit of the identification block. uint64_t BitcodeStartBit; @@ -119,15 +130,22 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { /// Tracks the last value id recorded in the GUIDToValueMap. unsigned GlobalValueId; + /// Saves the offset of the VSTOffset record that must eventually be + /// backpatched with the offset of the actual VST. + uint64_t VSTOffsetPlaceholder = 0; + public: /// Constructs a ModuleBitcodeWriter object for the given Module, /// writing to the provided \p Buffer. ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, + StringTableBuilder &StrtabBuilder, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - const ModuleSummaryIndex *Index, bool GenerateHash) - : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M), - VE(*M, ShouldPreserveUseListOrder), Index(Index), - GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) { + const ModuleSummaryIndex *Index, bool GenerateHash, + ModuleHash *ModHash = nullptr) + : BitcodeWriterBase(Stream), Buffer(Buffer), StrtabBuilder(StrtabBuilder), + M(*M), VE(*M, ShouldPreserveUseListOrder), Index(Index), + GenerateHash(GenerateHash), ModHash(ModHash), + BitcodeStartBit(Stream.GetCurrentBitNo()) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). @@ -138,14 +156,14 @@ public: return; for (const auto &GUIDSummaryLists : *Index) // Examine all summaries for this GUID. - for (auto &Summary : GUIDSummaryLists.second) + for (auto &Summary : GUIDSummaryLists.second.SummaryList) if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) // For each call in the function summary, see if the call // is to a GUID (which means it is for an indirect call, // otherwise we would have a Value for it). If so, synthesize // a value id. for (auto &CallEdge : FS->calls()) - if (CallEdge.first.isGUID()) + if (!CallEdge.first.getValue()) assignValueId(CallEdge.first.getGUID()); } @@ -159,6 +177,7 @@ private: void writeAttributeTable(); void writeTypeTable(); void writeComdats(); + void writeValueSymbolTableForwardDecl(); void writeModuleInfo(); void writeValueAsMetadata(const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record); @@ -251,9 +270,9 @@ private: SmallVectorImpl<uint64_t> &Vals); void writeInstruction(const Instruction &I, unsigned InstID, SmallVectorImpl<unsigned> &Vals); - void writeValueSymbolTable( - const ValueSymbolTable &VST, bool IsModuleLevel = false, - DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex = nullptr); + void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST); + void writeGlobalValueSymbolTable( + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex); void writeUseList(UseListOrder &&Order); void writeUseListBlock(const Function *F); void @@ -285,7 +304,7 @@ private: } // Helper to get the valueId for the type of value recorded in VI. unsigned getValueId(ValueInfo VI) { - if (VI.isGUID()) + if (!VI.getValue()) return getValueId(VI.getGUID()); return VE.getValueID(VI.getValue()); } @@ -321,154 +340,34 @@ public: // in writing out the call graph edges. Save the mapping from GUID // to the new global value id to use when writing those edges, which // are currently saved in the index in terms of GUID. - for (const auto &I : *this) + forEachSummary([&](GVInfo I) { GUIDToValueIdMap[I.first] = ++GlobalValueId; + }); } /// The below iterator returns the GUID and associated summary. typedef std::pair<GlobalValue::GUID, GlobalValueSummary *> GVInfo; - /// Iterator over the value GUID and summaries to be written to bitcode, - /// hides the details of whether they are being pulled from the entire - /// index or just those in a provided ModuleToSummariesForIndex map. - class iterator - : public llvm::iterator_facade_base<iterator, std::forward_iterator_tag, - GVInfo> { - /// Enables access to parent class. - const IndexBitcodeWriter &Writer; - - // Iterators used when writing only those summaries in a provided - // ModuleToSummariesForIndex map: - - /// Points to the last element in outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesBack; - /// Iterator on outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesIter; - /// Iterator on an inner global variable summary map. - GVSummaryMapTy::const_iterator ModuleGVSummariesIter; - - // Iterators used when writing all summaries in the index: - - /// Points to the last element in the Index outer GlobalValueMap. - const_gvsummary_iterator IndexSummariesBack; - /// Iterator on outer GlobalValueMap. - const_gvsummary_iterator IndexSummariesIter; - /// Iterator on an inner GlobalValueSummaryList. - GlobalValueSummaryList::const_iterator IndexGVSummariesIter; - - public: - /// Construct iterator from parent \p Writer and indicate if we are - /// constructing the end iterator. - iterator(const IndexBitcodeWriter &Writer, bool IsAtEnd) : Writer(Writer) { - // Set up the appropriate set of iterators given whether we are writing - // the full index or just a subset. - // Can't setup the Back or inner iterators if the corresponding map - // is empty. This will be handled specially in operator== as well. - if (Writer.ModuleToSummariesForIndex && - !Writer.ModuleToSummariesForIndex->empty()) { - for (ModuleSummariesBack = Writer.ModuleToSummariesForIndex->begin(); - std::next(ModuleSummariesBack) != - Writer.ModuleToSummariesForIndex->end(); - ModuleSummariesBack++) - ; - ModuleSummariesIter = !IsAtEnd - ? Writer.ModuleToSummariesForIndex->begin() - : ModuleSummariesBack; - ModuleGVSummariesIter = !IsAtEnd ? ModuleSummariesIter->second.begin() - : ModuleSummariesBack->second.end(); - } else if (!Writer.ModuleToSummariesForIndex && - Writer.Index.begin() != Writer.Index.end()) { - for (IndexSummariesBack = Writer.Index.begin(); - std::next(IndexSummariesBack) != Writer.Index.end(); - IndexSummariesBack++) - ; - IndexSummariesIter = - !IsAtEnd ? Writer.Index.begin() : IndexSummariesBack; - IndexGVSummariesIter = !IsAtEnd ? IndexSummariesIter->second.begin() - : IndexSummariesBack->second.end(); - } - } - - /// Increment the appropriate set of iterators. - iterator &operator++() { - // First the inner iterator is incremented, then if it is at the end - // and there are more outer iterations to go, the inner is reset to - // the start of the next inner list. - if (Writer.ModuleToSummariesForIndex) { - ++ModuleGVSummariesIter; - if (ModuleGVSummariesIter == ModuleSummariesIter->second.end() && - ModuleSummariesIter != ModuleSummariesBack) { - ++ModuleSummariesIter; - ModuleGVSummariesIter = ModuleSummariesIter->second.begin(); - } - } else { - ++IndexGVSummariesIter; - if (IndexGVSummariesIter == IndexSummariesIter->second.end() && - IndexSummariesIter != IndexSummariesBack) { - ++IndexSummariesIter; - IndexGVSummariesIter = IndexSummariesIter->second.begin(); - } - } - return *this; - } - - /// Access the <GUID,GlobalValueSummary*> pair corresponding to the current - /// outer and inner iterator positions. - GVInfo operator*() { - if (Writer.ModuleToSummariesForIndex) - return std::make_pair(ModuleGVSummariesIter->first, - ModuleGVSummariesIter->second); - return std::make_pair(IndexSummariesIter->first, - IndexGVSummariesIter->get()); - } - - /// Checks if the iterators are equal, with special handling for empty - /// indexes. - bool operator==(const iterator &RHS) const { - if (Writer.ModuleToSummariesForIndex) { - // First ensure that both are writing the same subset. - if (Writer.ModuleToSummariesForIndex != - RHS.Writer.ModuleToSummariesForIndex) - return false; - // Already determined above that maps are the same, so if one is - // empty, they both are. - if (Writer.ModuleToSummariesForIndex->empty()) - return true; - // Ensure the ModuleGVSummariesIter are iterating over the same - // container before checking them below. - if (ModuleSummariesIter != RHS.ModuleSummariesIter) - return false; - return ModuleGVSummariesIter == RHS.ModuleGVSummariesIter; - } - // First ensure RHS also writing the full index, and that both are - // writing the same full index. - if (RHS.Writer.ModuleToSummariesForIndex || - &Writer.Index != &RHS.Writer.Index) - return false; - // Already determined above that maps are the same, so if one is - // empty, they both are. - if (Writer.Index.begin() == Writer.Index.end()) - return true; - // Ensure the IndexGVSummariesIter are iterating over the same - // container before checking them below. - if (IndexSummariesIter != RHS.IndexSummariesIter) - return false; - return IndexGVSummariesIter == RHS.IndexGVSummariesIter; + /// Calls the callback for each value GUID and summary to be written to + /// bitcode. This hides the details of whether they are being pulled from the + /// entire index or just those in a provided ModuleToSummariesForIndex map. + void forEachSummary(std::function<void(GVInfo)> Callback) { + if (ModuleToSummariesForIndex) { + for (auto &M : *ModuleToSummariesForIndex) + for (auto &Summary : M.second) + Callback(Summary); + } else { + for (auto &Summaries : Index) + for (auto &Summary : Summaries.second.SummaryList) + Callback({Summaries.first, Summary.get()}); } - }; - - /// Obtain the start iterator over the summaries to be written. - iterator begin() { return iterator(*this, /*IsAtEnd=*/false); } - /// Obtain the end iterator over the summaries to be written. - iterator end() { return iterator(*this, /*IsAtEnd=*/true); } + } /// Main entry point for writing a combined index to bitcode. void write(); private: - void writeIndex(); void writeModStrings(); - void writeCombinedValueSymbolTable(); void writeCombinedGlobalValueSummary(); /// Indicates whether the provided \p ModulePath should be written into @@ -479,19 +378,11 @@ private: ModuleToSummariesForIndex->count(ModulePath); } - bool hasValueId(GlobalValue::GUID ValGUID) { - const auto &VMI = GUIDToValueIdMap.find(ValGUID); - return VMI != GUIDToValueIdMap.end(); - } - unsigned getValueId(GlobalValue::GUID ValGUID) { - const auto &VMI = GUIDToValueIdMap.find(ValGUID); - // If this GUID doesn't have an entry, assign one. - if (VMI == GUIDToValueIdMap.end()) { - GUIDToValueIdMap[ValGUID] = ++GlobalValueId; - return GlobalValueId; - } else { - return VMI->second; - } + Optional<unsigned> getValueId(GlobalValue::GUID ValGUID) { + auto VMI = GUIDToValueIdMap.find(ValGUID); + if (VMI == GUIDToValueIdMap.end()) + return None; + return VMI->second; } std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } }; @@ -671,6 +562,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_RETURNS_TWICE; case Attribute::SExt: return bitc::ATTR_KIND_S_EXT; + case Attribute::Speculatable: + return bitc::ATTR_KIND_SPECULATABLE; case Attribute::StackAlignment: return bitc::ATTR_KIND_STACK_ALIGNMENT; case Attribute::StackProtect: @@ -709,63 +602,62 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { } void ModuleBitcodeWriter::writeAttributeGroupTable() { - const std::vector<AttributeSet> &AttrGrps = VE.getAttributeGroups(); + const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps = + VE.getAttributeGroups(); if (AttrGrps.empty()) return; Stream.EnterSubblock(bitc::PARAMATTR_GROUP_BLOCK_ID, 3); SmallVector<uint64_t, 64> Record; - for (unsigned i = 0, e = AttrGrps.size(); i != e; ++i) { - AttributeSet AS = AttrGrps[i]; - for (unsigned i = 0, e = AS.getNumSlots(); i != e; ++i) { - AttributeSet A = AS.getSlotAttributes(i); - - Record.push_back(VE.getAttributeGroupID(A)); - Record.push_back(AS.getSlotIndex(i)); - - for (AttributeSet::iterator I = AS.begin(0), E = AS.end(0); - I != E; ++I) { - Attribute Attr = *I; - if (Attr.isEnumAttribute()) { - Record.push_back(0); - Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); - } else if (Attr.isIntAttribute()) { - Record.push_back(1); - Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); - Record.push_back(Attr.getValueAsInt()); - } else { - StringRef Kind = Attr.getKindAsString(); - StringRef Val = Attr.getValueAsString(); - - Record.push_back(Val.empty() ? 3 : 4); - Record.append(Kind.begin(), Kind.end()); + for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) { + unsigned AttrListIndex = Pair.first; + AttributeSet AS = Pair.second; + Record.push_back(VE.getAttributeGroupID(Pair)); + Record.push_back(AttrListIndex); + + for (Attribute Attr : AS) { + if (Attr.isEnumAttribute()) { + Record.push_back(0); + Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); + } else if (Attr.isIntAttribute()) { + Record.push_back(1); + Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); + Record.push_back(Attr.getValueAsInt()); + } else { + StringRef Kind = Attr.getKindAsString(); + StringRef Val = Attr.getValueAsString(); + + Record.push_back(Val.empty() ? 3 : 4); + Record.append(Kind.begin(), Kind.end()); + Record.push_back(0); + if (!Val.empty()) { + Record.append(Val.begin(), Val.end()); Record.push_back(0); - if (!Val.empty()) { - Record.append(Val.begin(), Val.end()); - Record.push_back(0); - } } } - - Stream.EmitRecord(bitc::PARAMATTR_GRP_CODE_ENTRY, Record); - Record.clear(); } + + Stream.EmitRecord(bitc::PARAMATTR_GRP_CODE_ENTRY, Record); + Record.clear(); } Stream.ExitBlock(); } void ModuleBitcodeWriter::writeAttributeTable() { - const std::vector<AttributeSet> &Attrs = VE.getAttributes(); + const std::vector<AttributeList> &Attrs = VE.getAttributeLists(); if (Attrs.empty()) return; Stream.EnterSubblock(bitc::PARAMATTR_BLOCK_ID, 3); SmallVector<uint64_t, 64> Record; for (unsigned i = 0, e = Attrs.size(); i != e; ++i) { - const AttributeSet &A = Attrs[i]; - for (unsigned i = 0, e = A.getNumSlots(); i != e; ++i) - Record.push_back(VE.getAttributeGroupID(A.getSlotAttributes(i))); + AttributeList AL = Attrs[i]; + for (unsigned i = AL.index_begin(), e = AL.index_end(); i != e; ++i) { + AttributeSet AS = AL.getAttributes(i); + if (AS.hasAttributes()) + Record.push_back(VE.getAttributeGroupID({i, AS})); + } Stream.EmitRecord(bitc::PARAMATTR_CODE_ENTRY, Record); Record.clear(); @@ -1038,13 +930,10 @@ static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) { void ModuleBitcodeWriter::writeComdats() { SmallVector<unsigned, 64> Vals; for (const Comdat *C : VE.getComdats()) { - // COMDAT: [selection_kind, name] + // COMDAT: [strtab offset, strtab size, selection_kind] + Vals.push_back(StrtabBuilder.add(C->getName())); + Vals.push_back(C->getName().size()); Vals.push_back(getEncodedComdatSelectionKind(*C)); - size_t Size = C->getName().size(); - assert(isUInt<32>(Size)); - Vals.push_back(Size); - for (char Chr : C->getName()) - Vals.push_back((unsigned char)Chr); Stream.EmitRecord(bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/0); Vals.clear(); } @@ -1053,7 +942,7 @@ void ModuleBitcodeWriter::writeComdats() { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched /// after the real VST is written. Saves the bit offset to backpatch. -void BitcodeWriterBase::writeValueSymbolTableForwardDecl() { +void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be @@ -1156,6 +1045,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Add an abbrev for common globals with no visibility or thread localness. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(MaxGlobalType+1))); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2 @@ -1179,15 +1070,42 @@ void ModuleBitcodeWriter::writeModuleInfo() { SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } - // Emit the global variable information. SmallVector<unsigned, 64> Vals; + // Emit the module's source file name. + { + StringEncoding Bits = getStringEncoding(M.getSourceFileName().data(), + M.getSourceFileName().size()); + BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); + if (Bits == SE_Char6) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); + else if (Bits == SE_Fixed7) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); + + // MODULE_CODE_SOURCE_FILENAME: [namechar x N] + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(AbbrevOpToUse); + unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + for (const auto P : M.getSourceFileName()) + Vals.push_back((unsigned char)P); + + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); + Vals.clear(); + } + + // Emit the global variable information. for (const GlobalVariable &GV : M.globals()) { unsigned AbbrevToUse = 0; - // GLOBALVAR: [type, isconst, initid, + // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid, // linkage, alignment, section, visibility, threadlocal, // unnamed_addr, externally_initialized, dllstorageclass, - // comdat] + // comdat, attributes] + Vals.push_back(StrtabBuilder.add(GV.getName())); + Vals.push_back(GV.getName().size()); Vals.push_back(VE.getTypeID(GV.getValueType())); Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant()); Vals.push_back(GV.isDeclaration() ? 0 : @@ -1200,13 +1118,17 @@ void ModuleBitcodeWriter::writeModuleInfo() { GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None || GV.isExternallyInitialized() || GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass || - GV.hasComdat()) { + GV.hasComdat() || + GV.hasAttributes()) { Vals.push_back(getEncodedVisibility(GV)); Vals.push_back(getEncodedThreadLocalMode(GV)); Vals.push_back(getEncodedUnnamedAddr(GV)); Vals.push_back(GV.isExternallyInitialized()); Vals.push_back(getEncodedDLLStorageClass(GV)); Vals.push_back(GV.hasComdat() ? VE.getComdatID(GV.getComdat()) : 0); + + auto AL = GV.getAttributesAsList(AttributeList::FunctionIndex); + Vals.push_back(VE.getAttributeListID(AL)); } else { AbbrevToUse = SimpleGVarAbbrev; } @@ -1217,14 +1139,17 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the function proto information. for (const Function &F : M) { - // FUNCTION: [type, callingconv, isproto, linkage, paramattrs, alignment, - // section, visibility, gc, unnamed_addr, prologuedata, - // dllstorageclass, comdat, prefixdata, personalityfn] + // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto, + // linkage, paramattrs, alignment, section, visibility, gc, + // unnamed_addr, prologuedata, dllstorageclass, comdat, + // prefixdata, personalityfn] + Vals.push_back(StrtabBuilder.add(F.getName())); + Vals.push_back(F.getName().size()); Vals.push_back(VE.getTypeID(F.getFunctionType())); Vals.push_back(F.getCallingConv()); Vals.push_back(F.isDeclaration()); Vals.push_back(getEncodedLinkage(F)); - Vals.push_back(VE.getAttributeID(F.getAttributes())); + Vals.push_back(VE.getAttributeListID(F.getAttributes())); Vals.push_back(Log2_32(F.getAlignment())+1); Vals.push_back(F.hasSection() ? SectionMap[F.getSection()] : 0); Vals.push_back(getEncodedVisibility(F)); @@ -1246,8 +1171,10 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the alias information. for (const GlobalAlias &A : M.aliases()) { - // ALIAS: [alias type, aliasee val#, linkage, visibility, dllstorageclass, - // threadlocal, unnamed_addr] + // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage, + // visibility, dllstorageclass, threadlocal, unnamed_addr] + Vals.push_back(StrtabBuilder.add(A.getName())); + Vals.push_back(A.getName().size()); Vals.push_back(VE.getTypeID(A.getValueType())); Vals.push_back(A.getType()->getAddressSpace()); Vals.push_back(VE.getValueID(A.getAliasee())); @@ -1263,7 +1190,10 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the ifunc information. for (const GlobalIFunc &I : M.ifuncs()) { - // IFUNC: [ifunc type, address space, resolver val#, linkage, visibility] + // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver + // val#, linkage, visibility] + Vals.push_back(StrtabBuilder.add(I.getName())); + Vals.push_back(I.getName().size()); Vals.push_back(VE.getTypeID(I.getValueType())); Vals.push_back(I.getType()->getAddressSpace()); Vals.push_back(VE.getValueID(I.getResolver())); @@ -1273,34 +1203,6 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.clear(); } - // Emit the module's source file name. - { - StringEncoding Bits = getStringEncoding(M.getSourceFileName().data(), - M.getSourceFileName().size()); - BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); - if (Bits == SE_Char6) - AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); - else if (Bits == SE_Fixed7) - AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); - - // MODULE_CODE_SOURCE_FILENAME: [namechar x N] - auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(AbbrevOpToUse); - unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - - for (const auto P : M.getSourceFileName()) - Vals.push_back((unsigned char)P); - - // Emit the finished record. - Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); - Vals.clear(); - } - - // If we have a VST, write the VSTOFFSET record placeholder. - if (M.getValueSymbolTable().empty()) - return; writeValueSymbolTableForwardDecl(); } @@ -1326,6 +1228,8 @@ static uint64_t getOptimizationFlags(const Value *V) { Flags |= FastMathFlags::NoSignedZeros; if (FPMO->hasAllowReciprocal()) Flags |= FastMathFlags::AllowReciprocal; + if (FPMO->hasAllowContract()) + Flags |= FastMathFlags::AllowContract; } return Flags; @@ -1473,6 +1377,13 @@ void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N, Record.push_back(N->getFlags()); Record.push_back(VE.getMetadataOrNullID(N->getExtraData())); + // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means + // that there is no DWARF address space associated with DIDerivedType. + if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace()) + Record.push_back(*DWARFAddressSpace + 1); + else + Record.push_back(0); + Stream.EmitRecord(bitc::METADATA_DERIVED_TYPE, Record, Abbrev); Record.clear(); } @@ -1549,6 +1460,7 @@ void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, Record.push_back(N->getDWOId()); Record.push_back(VE.getMetadataOrNullID(N->getMacros().get())); Record.push_back(N->getSplitDebugInlining()); + Record.push_back(N->getDebugInfoForProfiling()); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -1578,6 +1490,7 @@ void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N, Record.push_back(VE.getMetadataOrNullID(N->getDeclaration())); Record.push_back(VE.getMetadataOrNullID(N->getVariables().get())); Record.push_back(N->getThisAdjustment()); + Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get())); Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev); Record.clear(); @@ -1613,9 +1526,7 @@ void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, unsigned Abbrev) { Record.push_back(N->isDistinct() | N->getExportSymbols() << 1); Record.push_back(VE.getMetadataOrNullID(N->getScope())); - Record.push_back(VE.getMetadataOrNullID(N->getFile())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); - Record.push_back(N->getLine()); Stream.EmitRecord(bitc::METADATA_NAMESPACE, Record, Abbrev); Record.clear(); @@ -1738,9 +1649,8 @@ void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { Record.reserve(N->getElements().size() + 1); - - const uint64_t HasOpFragmentFlag = 1 << 1; - Record.push_back((uint64_t)N->isDistinct() | HasOpFragmentFlag); + const uint64_t Version = 2 << 1; + Record.push_back((uint64_t)N->isDistinct() | Version); Record.append(N->elements_begin(), N->elements_end()); Stream.EmitRecord(bitc::METADATA_EXPRESSION, Record, Abbrev); @@ -2559,7 +2469,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(VE.getTypeID(SI.getCondition()->getType())); pushValue(SI.getCondition(), InstID, Vals); Vals.push_back(VE.getValueID(SI.getDefaultDest())); - for (SwitchInst::ConstCaseIt Case : SI.cases()) { + for (auto Case : SI.cases()) { Vals.push_back(VE.getValueID(Case.getCaseValue())); Vals.push_back(VE.getValueID(Case.getCaseSuccessor())); } @@ -2584,7 +2494,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_INVOKE; - Vals.push_back(VE.getAttributeID(II->getAttributes())); + Vals.push_back(VE.getAttributeListID(II->getAttributes())); Vals.push_back(II->getCallingConv() | 1 << 13); Vals.push_back(VE.getValueID(II->getNormalDest())); Vals.push_back(VE.getValueID(II->getUnwindDest())); @@ -2776,7 +2686,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_CALL; - Vals.push_back(VE.getAttributeID(CI.getAttributes())); + Vals.push_back(VE.getAttributeListID(CI.getAttributes())); unsigned Flags = getOptimizationFlags(&I); Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV | @@ -2820,77 +2730,59 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.clear(); } -/// Emit names for globals/functions etc. \p IsModuleLevel is true when -/// we are writing the module-level VST, where we are including a function -/// bitcode index and need to backpatch the VST forward declaration record. -void ModuleBitcodeWriter::writeValueSymbolTable( - const ValueSymbolTable &VST, bool IsModuleLevel, - DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex) { - if (VST.empty()) { - // writeValueSymbolTableForwardDecl should have returned early as - // well. Ensure this handling remains in sync by asserting that - // the placeholder offset is not set. - assert(!IsModuleLevel || !hasVSTOffsetPlaceholder()); - return; - } +/// Write a GlobalValue VST to the module. The purpose of this data structure is +/// to allow clients to efficiently find the function body. +void ModuleBitcodeWriter::writeGlobalValueSymbolTable( + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) { + // Get the offset of the VST we are writing, and backpatch it into + // the VST forward declaration record. + uint64_t VSTOffset = Stream.GetCurrentBitNo(); + // The BitcodeStartBit was the stream offset of the identification block. + VSTOffset -= bitcodeStartBit(); + assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + // Note that we add 1 here because the offset is relative to one word + // before the start of the identification block, which was historically + // always the start of the regular bitcode header. + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1); + + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); + + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset + unsigned FnEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + for (const Function &F : M) { + uint64_t Record[2]; + + if (F.isDeclaration()) + continue; + + Record[0] = VE.getValueID(&F); - if (IsModuleLevel && hasVSTOffsetPlaceholder()) { - // Get the offset of the VST we are writing, and backpatch it into - // the VST forward declaration record. - uint64_t VSTOffset = Stream.GetCurrentBitNo(); - // The BitcodeStartBit was the stream offset of the identification block. - VSTOffset -= bitcodeStartBit(); - assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + // Save the word offset of the function (from the start of the + // actual bitcode written to the stream). + uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit(); + assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); // Note that we add 1 here because the offset is relative to one word // before the start of the identification block, which was historically // always the start of the regular bitcode header. - Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1); - } - - Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); + Record[1] = BitcodeIndex / 32 + 1; - // For the module-level VST, add abbrev Ids for the VST_CODE_FNENTRY - // records, which are not used in the per-function VSTs. - unsigned FnEntry8BitAbbrev; - unsigned FnEntry7BitAbbrev; - unsigned FnEntry6BitAbbrev; - unsigned GUIDEntryAbbrev; - if (IsModuleLevel && hasVSTOffsetPlaceholder()) { - // 8-bit fixed-width VST_CODE_FNENTRY function strings. - auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - FnEntry8BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + Stream.EmitRecord(bitc::VST_CODE_FNENTRY, Record, FnEntryAbbrev); + } - // 7-bit fixed width VST_CODE_FNENTRY function strings. - Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - FnEntry7BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + Stream.ExitBlock(); +} - // 6-bit char6 VST_CODE_FNENTRY function strings. - Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - FnEntry6BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); +/// Emit names for arguments, instructions and basic blocks in a function. +void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable( + const ValueSymbolTable &VST) { + if (VST.empty()) + return; - // FIXME: Change the name of this record as it is now used by - // the per-module index as well. - Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - GUIDEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - } + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); // FIXME: Set up the abbrev, we know how many values there are! // FIXME: We know if the type names can use 7-bit ascii. @@ -2904,45 +2796,13 @@ void ModuleBitcodeWriter::writeValueSymbolTable( unsigned AbbrevToUse = VST_ENTRY_8_ABBREV; NameVals.push_back(VE.getValueID(Name.getValue())); - Function *F = dyn_cast<Function>(Name.getValue()); - if (!F) { - // If value is an alias, need to get the aliased base object to - // see if it is a function. - auto *GA = dyn_cast<GlobalAlias>(Name.getValue()); - if (GA && GA->getBaseObject()) - F = dyn_cast<Function>(GA->getBaseObject()); - } - // VST_CODE_ENTRY: [valueid, namechar x N] - // VST_CODE_FNENTRY: [valueid, funcoffset, namechar x N] // VST_CODE_BBENTRY: [bbid, namechar x N] unsigned Code; if (isa<BasicBlock>(Name.getValue())) { Code = bitc::VST_CODE_BBENTRY; if (Bits == SE_Char6) AbbrevToUse = VST_BBENTRY_6_ABBREV; - } else if (F && !F->isDeclaration()) { - // Must be the module-level VST, where we pass in the Index and - // have a VSTOffsetPlaceholder. The function-level VST should not - // contain any Function symbols. - assert(FunctionToBitcodeIndex); - assert(hasVSTOffsetPlaceholder()); - - // Save the word offset of the function (from the start of the - // actual bitcode written to the stream). - uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - bitcodeStartBit(); - assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); - // Note that we add 1 here because the offset is relative to one word - // before the start of the identification block, which was historically - // always the start of the regular bitcode header. - NameVals.push_back(BitcodeIndex / 32 + 1); - - Code = bitc::VST_CODE_FNENTRY; - AbbrevToUse = FnEntry8BitAbbrev; - if (Bits == SE_Char6) - AbbrevToUse = FnEntry6BitAbbrev; - else if (Bits == SE_Fixed7) - AbbrevToUse = FnEntry7BitAbbrev; } else { Code = bitc::VST_CODE_ENTRY; if (Bits == SE_Char6) @@ -2958,47 +2818,7 @@ void ModuleBitcodeWriter::writeValueSymbolTable( Stream.EmitRecord(Code, NameVals, AbbrevToUse); NameVals.clear(); } - // Emit any GUID valueIDs created for indirect call edges into the - // module-level VST. - if (IsModuleLevel && hasVSTOffsetPlaceholder()) - for (const auto &GI : valueIds()) { - NameVals.push_back(GI.second); - NameVals.push_back(GI.first); - Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, - GUIDEntryAbbrev); - NameVals.clear(); - } - Stream.ExitBlock(); -} - -/// Emit function names and summary offsets for the combined index -/// used by ThinLTO. -void IndexBitcodeWriter::writeCombinedValueSymbolTable() { - assert(hasVSTOffsetPlaceholder() && "Expected non-zero VSTOffsetPlaceholder"); - // Get the offset of the VST we are writing, and backpatch it into - // the VST forward declaration record. - uint64_t VSTOffset = Stream.GetCurrentBitNo(); - assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); - Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); - - Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); - - auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - unsigned EntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - - SmallVector<uint64_t, 64> NameVals; - for (const auto &GVI : valueIds()) { - // VST_CODE_COMBINED_ENTRY: [valueid, refguid] - NameVals.push_back(GVI.second); - NameVals.push_back(GVI.first); - // Emit the finished record. - Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev); - NameVals.clear(); - } Stream.ExitBlock(); } @@ -3102,7 +2922,7 @@ void ModuleBitcodeWriter::writeFunction( // Emit names for all the instructions etc. if (auto *Symtab = F.getValueSymbolTable()) - writeValueSymbolTable(*Symtab); + writeFunctionLevelValueSymbolTable(*Symtab); if (NeedsMetadataAttachment) writeFunctionMetadataAttachment(F); @@ -3367,6 +3187,49 @@ void IndexBitcodeWriter::writeModStrings() { Stream.ExitBlock(); } +/// Write the function type metadata related records that need to appear before +/// a function summary entry (whether per-module or combined). +static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, + FunctionSummary *FS) { + if (!FS->type_tests().empty()) + Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + + SmallVector<uint64_t, 64> Record; + + auto WriteVFuncIdVec = [&](uint64_t Ty, + ArrayRef<FunctionSummary::VFuncId> VFs) { + if (VFs.empty()) + return; + Record.clear(); + for (auto &VF : VFs) { + Record.push_back(VF.GUID); + Record.push_back(VF.Offset); + } + Stream.EmitRecord(Ty, Record); + }; + + WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS, + FS->type_test_assume_vcalls()); + WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS, + FS->type_checked_load_vcalls()); + + auto WriteConstVCallVec = [&](uint64_t Ty, + ArrayRef<FunctionSummary::ConstVCall> VCs) { + for (auto &VC : VCs) { + Record.clear(); + Record.push_back(VC.VFunc.GUID); + Record.push_back(VC.VFunc.Offset); + Record.insert(Record.end(), VC.Args.begin(), VC.Args.end()); + Stream.EmitRecord(Ty, Record); + } + }; + + WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL, + FS->type_test_assume_const_vcalls()); + WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL, + FS->type_checked_load_const_vcalls()); +} + // Helper to emit a single function summary record. void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, @@ -3375,8 +3238,7 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( NameVals.push_back(ValueID); FunctionSummary *FS = cast<FunctionSummary>(Summary); - if (!FS->type_tests().empty()) - Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + writeFunctionTypeMetadataRecords(Stream, FS); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); @@ -3406,15 +3268,14 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( void ModuleBitcodeWriter::writeModuleLevelReferences( const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev) { - auto Summaries = - Index->findGlobalValueSummaryList(GlobalValue::getGUID(V.getName())); - if (Summaries == Index->end()) { + auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName())); + if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might however // have a summary if the def was in module level asm). assert(V.isDeclaration()); return; } - auto *Summary = Summaries->second.front().get(); + auto *Summary = VI.getSummaryList()[0].get(); NameVals.push_back(VE.getValueID(&V)); GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); @@ -3448,6 +3309,11 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { return; } + for (const auto &GVI : valueIds()) { + Stream.EmitRecord(bitc::FS_VALUE_GUID, + ArrayRef<uint64_t>{GVI.second, GVI.first}); + } + // Abbrev for FS_PERMODULE. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); @@ -3498,15 +3364,14 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - auto Summaries = - Index->findGlobalValueSummaryList(GlobalValue::getGUID(F.getName())); - if (Summaries == Index->end()) { + ValueInfo VI = Index->getValueInfo(GlobalValue::getGUID(F.getName())); + if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might // however have a summary if the def was in module level asm). assert(F.isDeclaration()); continue; } - auto *Summary = Summaries->second.front().get(); + auto *Summary = VI.getSummaryList()[0].get(); writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), FSCallsAbbrev, FSCallsProfileAbbrev, F); } @@ -3540,6 +3405,11 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + for (const auto &GVI : valueIds()) { + Stream.EmitRecord(bitc::FS_VALUE_GUID, + ArrayRef<uint64_t>{GVI.second, GVI.first}); + } + // Abbrev for FS_COMBINED. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); @@ -3604,27 +3474,30 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.clear(); }; - for (const auto &I : *this) { + forEachSummary([&](GVInfo I) { GlobalValueSummary *S = I.second; assert(S); - assert(hasValueId(I.first)); - unsigned ValueId = getValueId(I.first); - SummaryToValueIdMap[S] = ValueId; + auto ValueId = getValueId(I.first); + assert(ValueId); + SummaryToValueIdMap[S] = *ValueId; if (auto *AS = dyn_cast<AliasSummary>(S)) { // Will process aliases as a post-pass because the reader wants all // global to be loaded first. Aliases.push_back(AS); - continue; + return; } if (auto *VS = dyn_cast<GlobalVarSummary>(S)) { - NameVals.push_back(ValueId); + NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(VS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); for (auto &RI : VS->refs()) { - NameVals.push_back(getValueId(RI.getGUID())); + auto RefValueId = getValueId(RI.getGUID()); + if (!RefValueId) + continue; + NameVals.push_back(*RefValueId); } // Emit the finished record. @@ -3632,22 +3505,28 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { FSModRefsAbbrev); NameVals.clear(); MaybeEmitOriginalName(*S); - continue; + return; } auto *FS = cast<FunctionSummary>(S); - if (!FS->type_tests().empty()) - Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + writeFunctionTypeMetadataRecords(Stream, FS); - NameVals.push_back(ValueId); + NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); - NameVals.push_back(FS->refs().size()); + // Fill in below + NameVals.push_back(0); + unsigned Count = 0; for (auto &RI : FS->refs()) { - NameVals.push_back(getValueId(RI.getGUID())); + auto RefValueId = getValueId(RI.getGUID()); + if (!RefValueId) + continue; + NameVals.push_back(*RefValueId); + Count++; } + NameVals[4] = Count; bool HasProfileData = false; for (auto &EI : FS->calls()) { @@ -3659,9 +3538,20 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { for (auto &EI : FS->calls()) { // If this GUID doesn't have a value id, it doesn't have a function // summary and we don't need to record any calls to it. - if (!hasValueId(EI.first.getGUID())) - continue; - NameVals.push_back(getValueId(EI.first.getGUID())); + GlobalValue::GUID GUID = EI.first.getGUID(); + auto CallValueId = getValueId(GUID); + if (!CallValueId) { + // For SamplePGO, the indirect call targets for local functions will + // have its original name annotated in profile. We try to find the + // corresponding PGOFuncName as the GUID. + GUID = Index.getGUIDFromOriginalID(GUID); + if (GUID == 0) + continue; + CallValueId = getValueId(GUID); + if (!CallValueId) + continue; + } + NameVals.push_back(*CallValueId); if (HasProfileData) NameVals.push_back(static_cast<uint8_t>(EI.second.Hotness)); } @@ -3674,7 +3564,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EmitRecord(Code, NameVals, FSAbbrev); NameVals.clear(); MaybeEmitOriginalName(*S); - } + }); for (auto *AS : Aliases) { auto AliasValueId = SummaryToValueIdMap[AS]; @@ -3697,7 +3587,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the /// current llvm version, and a record for the epoch number. -void writeIdentificationBlock(BitstreamWriter &Stream) { +static void writeIdentificationBlock(BitstreamWriter &Stream) { Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5); // Write the "user readable" string identifying the bitcode producer @@ -3722,17 +3612,24 @@ void writeIdentificationBlock(BitstreamWriter &Stream) { void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { // Emit the module's hash. // MODULE_CODE_HASH: [5*i32] - SHA1 Hasher; - Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], - Buffer.size() - BlockStartPos)); - StringRef Hash = Hasher.result(); - uint32_t Vals[5]; - for (int Pos = 0; Pos < 20; Pos += 4) { - Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); - } + if (GenerateHash) { + SHA1 Hasher; + uint32_t Vals[5]; + Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], + Buffer.size() - BlockStartPos)); + StringRef Hash = Hasher.result(); + for (int Pos = 0; Pos < 20; Pos += 4) { + Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); + } - // Emit the finished record. - Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); + + if (ModHash) + // Save the written hash value. + std::copy(std::begin(Vals), std::end(Vals), std::begin(*ModHash)); + } else if (ModHash) + Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef<uint32_t>(*ModHash)); } void ModuleBitcodeWriter::write() { @@ -3741,10 +3638,7 @@ void ModuleBitcodeWriter::write() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); size_t BlockStartPos = Buffer.size(); - SmallVector<unsigned, 1> Vals; - unsigned CurVersion = 1; - Vals.push_back(CurVersion); - Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); + writeModuleVersion(); // Emit blockinfo, which defines the standard abbreviations etc. writeBlockInfo(); @@ -3790,12 +3684,9 @@ void ModuleBitcodeWriter::write() { if (Index) writePerModuleGlobalValueSummary(); - writeValueSymbolTable(M.getValueSymbolTable(), - /* IsModuleLevel */ true, &FunctionToBitcodeIndex); + writeGlobalValueSymbolTable(FunctionToBitcodeIndex); - if (GenerateHash) { - writeModuleHash(BlockStartPos); - } + writeModuleHash(BlockStartPos); Stream.ExitBlock(); } @@ -3881,14 +3772,47 @@ BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer) writeBitcodeHeader(*Stream); } -BitcodeWriter::~BitcodeWriter() = default; +BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); } + +void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) { + Stream->EnterSubblock(Block, 3); + + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(Record)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + auto AbbrevNo = Stream->EmitAbbrev(std::move(Abbv)); + + Stream->EmitRecordWithBlob(AbbrevNo, ArrayRef<uint64_t>{Record}, Blob); + + Stream->ExitBlock(); +} + +void BitcodeWriter::writeStrtab() { + assert(!WroteStrtab); + + std::vector<char> Strtab; + StrtabBuilder.finalizeInOrder(); + Strtab.resize(StrtabBuilder.getSize()); + StrtabBuilder.write((uint8_t *)Strtab.data()); + + writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB, + {Strtab.data(), Strtab.size()}); + + WroteStrtab = true; +} + +void BitcodeWriter::copyStrtab(StringRef Strtab) { + writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB, Strtab); + WroteStrtab = true; +} void BitcodeWriter::writeModule(const Module *M, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { - ModuleBitcodeWriter ModuleWriter( - M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash); + bool GenerateHash, ModuleHash *ModHash) { + ModuleBitcodeWriter ModuleWriter(M, Buffer, StrtabBuilder, *Stream, + ShouldPreserveUseListOrder, Index, + GenerateHash, ModHash); ModuleWriter.write(); } @@ -3897,7 +3821,7 @@ void BitcodeWriter::writeModule(const Module *M, void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { + bool GenerateHash, ModuleHash *ModHash) { SmallVector<char, 0> Buffer; Buffer.reserve(256*1024); @@ -3908,7 +3832,9 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); BitcodeWriter Writer(Buffer); - Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash); + Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash, + ModHash); + Writer.writeStrtab(); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) emitDarwinBCHeaderAndTrailer(Buffer, TT); @@ -3920,13 +3846,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, void IndexBitcodeWriter::write() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); - SmallVector<unsigned, 1> Vals; - unsigned CurVersion = 1; - Vals.push_back(CurVersion); - Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); - - // If we have a VST, write the VSTOFFSET record placeholder. - writeValueSymbolTableForwardDecl(); + writeModuleVersion(); // Write the module paths in the combined index. writeModStrings(); @@ -3934,10 +3854,6 @@ void IndexBitcodeWriter::write() { // Write the summary combined index records. writeCombinedGlobalValueSummary(); - // Need a special VST writer for the combined index (we don't have a - // real VST and real values when this is invoked). - writeCombinedValueSymbolTable(); - Stream.ExitBlock(); } |