diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2023-04-14 21:41:27 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2023-06-22 18:20:56 +0000 |
commit | bdd1243df58e60e85101c09001d9812a789b6bc4 (patch) | |
tree | a1ce621c7301dd47ba2ddc3b8eaa63b441389481 /contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | |
parent | 781624ca2d054430052c828ba8d2c2eaf2d733e7 (diff) | |
parent | e3b557809604d036af6e00c60f012c2025b59a5e (diff) |
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp')
-rw-r--r-- | contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 249 |
1 files changed, 220 insertions, 29 deletions
diff --git a/contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index d7e012fb6a9e..f85fd86c114c 100644 --- a/contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -16,8 +16,6 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/None.h" -#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" @@ -78,6 +76,7 @@ #include <iterator> #include <map> #include <memory> +#include <optional> #include <string> #include <utility> #include <vector> @@ -96,7 +95,9 @@ static cl::opt<bool> WriteRelBFToSummary( "write-relbf-to-summary", cl::Hidden, cl::init(false), cl::desc("Write relative block frequency to function summary ")); +namespace llvm { extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold; +} namespace { @@ -211,12 +212,10 @@ protected: void writePerModuleGlobalValueSummary(); private: - void writePerModuleFunctionSummaryRecord(SmallVector<uint64_t, 64> &NameVals, - GlobalValueSummary *Summary, - unsigned ValueID, - unsigned FSCallsAbbrev, - unsigned FSCallsProfileAbbrev, - const Function &F); + void writePerModuleFunctionSummaryRecord( + SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, + unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, + unsigned CallsiteAbbrev, unsigned AllocAbbrev, const Function &F); void writeModuleLevelReferences(const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev, @@ -340,6 +339,8 @@ private: unsigned Abbrev); void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); void writeDITemplateTypeParameter(const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); @@ -422,6 +423,11 @@ class IndexBitcodeWriter : public BitcodeWriterBase { /// index and a value id generated by this class to use in references. std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap; + // The sorted stack id indices actually used in the summary entries being + // written, which will be a subset of those in the full index in the case of + // distributed indexes. + std::vector<unsigned> StackIdIndices; + /// Tracks the last value id recorded in the GUIDToValueMap. unsigned GlobalValueId = 0; @@ -439,9 +445,28 @@ public: // in writing out the call graph edges. Save the mapping from GUID // to the new global value id to use when writing those edges, which // are currently saved in the index in terms of GUID. - forEachSummary([&](GVInfo I, bool) { + forEachSummary([&](GVInfo I, bool IsAliasee) { GUIDToValueIdMap[I.first] = ++GlobalValueId; + if (IsAliasee) + return; + auto *FS = dyn_cast<FunctionSummary>(I.second); + if (!FS) + return; + // Record all stack id indices actually used in the summary entries being + // written, so that we can compact them in the case of distributed ThinLTO + // indexes. + for (auto &CI : FS->callsites()) + for (auto Idx : CI.StackIdIndices) + StackIdIndices.push_back(Idx); + for (auto &AI : FS->allocs()) + for (auto &MIB : AI.MIBs) + for (auto Idx : MIB.StackIdIndices) + StackIdIndices.push_back(Idx); }); + llvm::sort(StackIdIndices); + StackIdIndices.erase( + std::unique(StackIdIndices.begin(), StackIdIndices.end()), + StackIdIndices.end()); } /// The below iterator returns the GUID and associated summary. @@ -499,10 +524,10 @@ private: void writeModStrings(); void writeCombinedGlobalValueSummary(); - Optional<unsigned> getValueId(GlobalValue::GUID ValGUID) { + std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) { auto VMI = GUIDToValueIdMap.find(ValGUID); if (VMI == GUIDToValueIdMap.end()) - return None; + return std::nullopt; return VMI->second; } @@ -579,6 +604,10 @@ static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) { case AtomicRMWInst::FSub: return bitc::RMW_FSUB; case AtomicRMWInst::FMax: return bitc::RMW_FMAX; case AtomicRMWInst::FMin: return bitc::RMW_FMIN; + case AtomicRMWInst::UIncWrap: + return bitc::RMW_UINC_WRAP; + case AtomicRMWInst::UDecWrap: + return bitc::RMW_UDEC_WRAP; } } @@ -620,8 +649,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_ALLOC_SIZE; case Attribute::AlwaysInline: return bitc::ATTR_KIND_ALWAYS_INLINE; - case Attribute::ArgMemOnly: - return bitc::ATTR_KIND_ARGMEMONLY; case Attribute::Builtin: return bitc::ATTR_KIND_BUILTIN; case Attribute::ByVal: @@ -640,10 +667,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_HOT; case Attribute::ElementType: return bitc::ATTR_KIND_ELEMENTTYPE; - case Attribute::InaccessibleMemOnly: - return bitc::ATTR_KIND_INACCESSIBLEMEM_ONLY; - case Attribute::InaccessibleMemOrArgMemOnly: - return bitc::ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY; case Attribute::InlineHint: return bitc::ATTR_KIND_INLINE_HINT; case Attribute::InReg: @@ -656,6 +679,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_ALLOCATED_POINTER; case Attribute::AllocKind: return bitc::ATTR_KIND_ALLOC_KIND; + case Attribute::Memory: + return bitc::ATTR_KIND_MEMORY; case Attribute::Naked: return bitc::ATTR_KIND_NAKED; case Attribute::Nest: @@ -698,6 +723,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NOCF_CHECK; case Attribute::NoProfile: return bitc::ATTR_KIND_NO_PROFILE; + case Attribute::SkipProfile: + return bitc::ATTR_KIND_SKIP_PROFILE; case Attribute::NoUnwind: return bitc::ATTR_KIND_NO_UNWIND; case Attribute::NoSanitizeBounds: @@ -1029,8 +1056,20 @@ void ModuleBitcodeWriter::writeTypeTable() { TypeVals.push_back(true); break; } - case Type::DXILPointerTyID: - llvm_unreachable("DXIL pointers cannot be added to IR modules"); + case Type::TargetExtTyID: { + TargetExtType *TET = cast<TargetExtType>(T); + Code = bitc::TYPE_CODE_TARGET_TYPE; + writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, TET->getName(), + StructNameAbbrev); + TypeVals.push_back(TET->getNumTypeParameters()); + for (Type *InnerTy : TET->type_params()) + TypeVals.push_back(VE.getTypeID(InnerTy)); + for (unsigned IntParam : TET->int_params()) + TypeVals.push_back(IntParam); + break; + } + case Type::TypedPointerTyID: + llvm_unreachable("Typed pointers cannot be added to IR modules"); } // Emit the finished record. @@ -1784,7 +1823,7 @@ void ModuleBitcodeWriter::writeDIFile(const DIFile *N, } auto Source = N->getRawSource(); if (Source) - Record.push_back(VE.getMetadataOrNullID(*Source)); + Record.push_back(VE.getMetadataOrNullID(Source)); Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev); Record.clear(); @@ -1951,6 +1990,15 @@ void ModuleBitcodeWriter::writeDIModule(const DIModule *N, Record.clear(); } +void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + // There are no arguments for this metadata type. + Record.push_back(N->isDistinct()); + Stream.EmitRecord(bitc::METADATA_ASSIGN_ID, Record, Abbrev); + Record.clear(); +} + void ModuleBitcodeWriter::writeDITemplateTypeParameter( const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { @@ -2624,7 +2672,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, Code = bitc::CST_CODE_CE_GEP; const auto *GO = cast<GEPOperator>(C); Record.push_back(VE.getTypeID(GO->getSourceElementType())); - if (Optional<unsigned> Idx = GO->getInRangeIndex()) { + if (std::optional<unsigned> Idx = GO->getInRangeIndex()) { Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX; Record.push_back((*Idx << 1) | GO->isInBounds()); } else if (GO->isInBounds()) @@ -2981,7 +3029,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, : bitc::FUNC_CODE_INST_CLEANUPPAD; pushValue(FuncletPad.getParentPad(), InstID, Vals); - unsigned NumArgOperands = FuncletPad.getNumArgOperands(); + unsigned NumArgOperands = FuncletPad.arg_size(); Vals.push_back(NumArgOperands); for (unsigned Op = 0; Op != NumArgOperands; ++Op) pushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals); @@ -3778,7 +3826,7 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, Record.push_back(Arg.Calls.size()); for (auto &Call : Arg.Calls) { Record.push_back(Call.ParamNo); - Optional<unsigned> ValueID = GetValueID(Call.Callee); + std::optional<unsigned> ValueID = GetValueID(Call.Callee); if (!ValueID) { // If ValueID is unknown we can't drop just this call, we must drop // entire parameter. @@ -3879,20 +3927,79 @@ static void writeTypeIdCompatibleVtableSummaryRecord( } } +static void writeFunctionHeapProfileRecords( + BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev, + unsigned AllocAbbrev, bool PerModule, + std::function<unsigned(const ValueInfo &VI)> GetValueID, + std::function<unsigned(unsigned)> GetStackIndex) { + SmallVector<uint64_t> Record; + + for (auto &CI : FS->callsites()) { + Record.clear(); + // Per module callsite clones should always have a single entry of + // value 0. + assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0)); + Record.push_back(GetValueID(CI.Callee)); + if (!PerModule) { + Record.push_back(CI.StackIdIndices.size()); + Record.push_back(CI.Clones.size()); + } + for (auto Id : CI.StackIdIndices) + Record.push_back(GetStackIndex(Id)); + if (!PerModule) { + for (auto V : CI.Clones) + Record.push_back(V); + } + Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO + : bitc::FS_COMBINED_CALLSITE_INFO, + Record, CallsiteAbbrev); + } + + for (auto &AI : FS->allocs()) { + Record.clear(); + // Per module alloc versions should always have a single entry of + // value 0. + assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0)); + if (!PerModule) { + Record.push_back(AI.MIBs.size()); + Record.push_back(AI.Versions.size()); + } + for (auto &MIB : AI.MIBs) { + Record.push_back((uint8_t)MIB.AllocType); + Record.push_back(MIB.StackIdIndices.size()); + for (auto Id : MIB.StackIdIndices) + Record.push_back(GetStackIndex(Id)); + } + if (!PerModule) { + for (auto V : AI.Versions) + Record.push_back(V); + } + Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_ALLOC_INFO + : bitc::FS_COMBINED_ALLOC_INFO, + Record, AllocAbbrev); + } +} + // Helper to emit a single function summary record. void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, - const Function &F) { + unsigned CallsiteAbbrev, unsigned AllocAbbrev, const Function &F) { NameVals.push_back(ValueID); FunctionSummary *FS = cast<FunctionSummary>(Summary); writeFunctionTypeMetadataRecords( - Stream, FS, [&](const ValueInfo &VI) -> Optional<unsigned> { + Stream, FS, [&](const ValueInfo &VI) -> std::optional<unsigned> { return {VE.getValueID(VI.getValue())}; }); + writeFunctionHeapProfileRecords( + Stream, FS, CallsiteAbbrev, AllocAbbrev, + /*PerModule*/ true, + /*GetValueId*/ [&](const ValueInfo &VI) { return getValueId(VI); }, + /*GetStackIndex*/ [&](unsigned I) { return I; }); + auto SpecialRefCnts = FS->specialRefCounts(); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); @@ -4004,6 +4111,16 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { ArrayRef<uint64_t>{GVI.second, GVI.first}); } + if (!Index->stackIds().empty()) { + auto StackIdAbbv = std::make_shared<BitCodeAbbrev>(); + StackIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_STACK_IDS)); + // numids x stackid + StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned StackIdAbbvId = Stream.EmitAbbrev(std::move(StackIdAbbv)); + Stream.EmitRecord(bitc::FS_STACK_IDS, Index->stackIds(), StackIdAbbvId); + } + // Abbrev for FS_PERMODULE_PROFILE. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); @@ -4075,6 +4192,21 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + // n x stackidindex + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned CallsiteAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO)); + // n x (alloc type, numstackids, numstackids x stackidindex) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + SmallVector<uint64_t, 64> NameVals; // Iterate over the list of functions instead of the Index to // ensure the ordering is stable. @@ -4093,7 +4225,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { } auto *Summary = VI.getSummaryList()[0].get(); writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), - FSCallsAbbrev, FSCallsProfileAbbrev, F); + FSCallsAbbrev, FSCallsProfileAbbrev, + CallsiteAbbrev, AllocAbbrev, F); } // Capture references from GlobalVariable initializers, which are outside @@ -4135,7 +4268,7 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { /// Emit the combined summary section into the combined index file. void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { - Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); Stream.EmitRecord( bitc::FS_VERSION, ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion}); @@ -4148,6 +4281,21 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { ArrayRef<uint64_t>{GVI.second, GVI.first}); } + if (!StackIdIndices.empty()) { + auto StackIdAbbv = std::make_shared<BitCodeAbbrev>(); + StackIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_STACK_IDS)); + // numids x stackid + StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned StackIdAbbvId = Stream.EmitAbbrev(std::move(StackIdAbbv)); + // Write the stack ids used by this index, which will be a subset of those in + // the full index in the case of distributed indexes. + std::vector<uint64_t> StackIds; + for (auto &I : StackIdIndices) + StackIds.push_back(Index.getStackIdAtIndex(I)); + Stream.EmitRecord(bitc::FS_STACK_IDS, StackIds, StackIdAbbvId); + } + // Abbrev for FS_COMBINED. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); @@ -4201,6 +4349,26 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver + // numstackindices x stackidindex, numver x version + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned CallsiteAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALLOC_INFO)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver + // nummib x (alloc type, numstackids, numstackids x stackidindex), + // numver x version + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // The aliases are emitted as a post-pass, and will point to the value // id of the aliasee. Save them in a vector for post-processing. SmallVector<AliasSummary *, 64> Aliases; @@ -4276,7 +4444,9 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { return; } - auto GetValueId = [&](const ValueInfo &VI) -> Optional<unsigned> { + auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> { + if (!VI) + return std::nullopt; return getValueId(VI.getGUID()); }; @@ -4284,6 +4454,27 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { writeFunctionTypeMetadataRecords(Stream, FS, GetValueId); getReferencedTypeIds(FS, ReferencedTypeIds); + writeFunctionHeapProfileRecords( + Stream, FS, CallsiteAbbrev, AllocAbbrev, + /*PerModule*/ false, + /*GetValueId*/ [&](const ValueInfo &VI) -> unsigned { + std::optional<unsigned> ValueID = GetValueId(VI); + // This can happen in shared index files for distributed ThinLTO if + // the callee function summary is not included. Record 0 which we + // will have to deal with conservatively when doing any kind of + // validation in the ThinLTO backends. + if (!ValueID) + return 0; + return *ValueID; + }, + /*GetStackIndex*/ [&](unsigned I) { + // Get the corresponding index into the list of StackIdIndices + // actually being written for this combined index (which may be a + // subset in the case of distributed indexes). + auto Lower = llvm::lower_bound(StackIdIndices, I); + return std::distance(StackIdIndices.begin(), Lower); + }); + NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); @@ -4323,7 +4514,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { for (auto &EI : FS->calls()) { // If this GUID doesn't have a value id, it doesn't have a function // summary and we don't need to record any calls to it. - Optional<unsigned> CallValueId = GetValueId(EI.first); + std::optional<unsigned> CallValueId = GetValueId(EI.first); if (!CallValueId) continue; NameVals.push_back(*CallValueId); |