diff options
Diffstat (limited to 'contrib/llvm/lib/Target/BPF')
45 files changed, 1937 insertions, 384 deletions
diff --git a/contrib/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/contrib/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 8890fb8adf4d..75885fd058a7 100644 --- a/contrib/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/contrib/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -1,13 +1,13 @@ //===-- BPFAsmParser.cpp - Parse BPF assembly to MCInst instructions --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "MCTargetDesc/BPFMCTargetDesc.h" +#include "TargetInfo/BPFTargetInfo.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/MC/MCContext.h" @@ -126,7 +126,7 @@ public: bool isMem() const override { return false; } bool isConstantImm() const { - return isImm() && dyn_cast<MCConstantExpr>(getImm()); + return isImm() && isa<MCConstantExpr>(getImm()); } int64_t getConstantImm() const { diff --git a/contrib/llvm/lib/Target/BPF/BPF.h b/contrib/llvm/lib/Target/BPF/BPF.h index 9749e369c2c1..d311fc154094 100644 --- a/contrib/llvm/lib/Target/BPF/BPF.h +++ b/contrib/llvm/lib/Target/BPF/BPF.h @@ -1,9 +1,8 @@ //===-- BPF.h - Top-level interface for BPF representation ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -16,11 +15,16 @@ namespace llvm { class BPFTargetMachine; +ModulePass *createBPFAbstractMemberAccess(); + FunctionPass *createBPFISelDag(BPFTargetMachine &TM); +FunctionPass *createBPFMISimplifyPatchablePass(); FunctionPass *createBPFMIPeepholePass(); FunctionPass *createBPFMIPreEmitPeepholePass(); FunctionPass *createBPFMIPreEmitCheckingPass(); +void initializeBPFAbstractMemberAccessPass(PassRegistry&); +void initializeBPFMISimplifyPatchablePass(PassRegistry&); void initializeBPFMIPeepholePass(PassRegistry&); void initializeBPFMIPreEmitPeepholePass(PassRegistry&); void initializeBPFMIPreEmitCheckingPass(PassRegistry&); diff --git a/contrib/llvm/lib/Target/BPF/BPF.td b/contrib/llvm/lib/Target/BPF/BPF.td index 877bd15f4f2b..fad966ff5a13 100644 --- a/contrib/llvm/lib/Target/BPF/BPF.td +++ b/contrib/llvm/lib/Target/BPF/BPF.td @@ -1,9 +1,8 @@ //===-- BPF.td - Describe the BPF Target Machine -----------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,6 +20,7 @@ class Proc<string Name, list<SubtargetFeature> Features> def : Proc<"generic", []>; def : Proc<"v1", []>; def : Proc<"v2", []>; +def : Proc<"v3", []>; def : Proc<"probe", []>; def DummyFeature : SubtargetFeature<"dummy", "isDummyMode", diff --git a/contrib/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/contrib/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp new file mode 100644 index 000000000000..51d4cbc8a429 --- /dev/null +++ b/contrib/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -0,0 +1,482 @@ +//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass abstracted struct/union member accesses in order to support +// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program +// which can run on different kernels. In particular, if bpf program tries to +// access a particular kernel data structure member, the details of the +// intermediate member access will be remembered so bpf loader can do +// necessary adjustment right before program loading. +// +// For example, +// +// struct s { +// int a; +// int b; +// }; +// struct t { +// struct s c; +// int d; +// }; +// struct t e; +// +// For the member access e.c.b, the compiler will generate code +// &e + 4 +// +// The compile-once run-everywhere instead generates the following code +// r = 4 +// &e + r +// The "4" in "r = 4" can be changed based on a particular kernel version. +// For example, on a particular kernel version, if struct s is changed to +// +// struct s { +// int new_field; +// int a; +// int b; +// } +// +// By repeating the member access on the host, the bpf loader can +// adjust "r = 4" as "r = 8". +// +// This feature relies on the following three intrinsic calls: +// addr = preserve_array_access_index(base, dimension, index) +// addr = preserve_union_access_index(base, di_index) +// !llvm.preserve.access.index <union_ditype> +// addr = preserve_struct_access_index(base, gep_index, di_index) +// !llvm.preserve.access.index <struct_ditype> +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include "BPFCORE.h" +#include "BPFTargetMachine.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/Pass.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" + +#define DEBUG_TYPE "bpf-abstract-member-access" + +namespace llvm { +const std::string BPFCoreSharedInfo::AmaAttr = "btf_ama"; +const std::string BPFCoreSharedInfo::PatchableExtSecName = + ".BPF.patchable_externs"; +} // namespace llvm + +using namespace llvm; + +namespace { + +class BPFAbstractMemberAccess final : public ModulePass { + StringRef getPassName() const override { + return "BPF Abstract Member Access"; + } + + bool runOnModule(Module &M) override; + +public: + static char ID; + BPFAbstractMemberAccess() : ModulePass(ID) {} + +private: + enum : uint32_t { + BPFPreserveArrayAI = 1, + BPFPreserveUnionAI = 2, + BPFPreserveStructAI = 3, + }; + + std::map<std::string, GlobalVariable *> GEPGlobals; + // A map to link preserve_*_access_index instrinsic calls. + std::map<CallInst *, std::pair<CallInst *, uint32_t>> AIChain; + // A map to hold all the base preserve_*_access_index instrinsic calls. + // The base call is not an input of any other preserve_*_access_index + // intrinsics. + std::map<CallInst *, uint32_t> BaseAICalls; + + bool doTransformation(Module &M); + + void traceAICall(CallInst *Call, uint32_t Kind); + void traceBitCast(BitCastInst *BitCast, CallInst *Parent, uint32_t Kind); + void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, uint32_t Kind); + void collectAICallChains(Module &M, Function &F); + + bool IsPreserveDIAccessIndexCall(const CallInst *Call, uint32_t &Kind); + bool removePreserveAccessIndexIntrinsic(Module &M); + void replaceWithGEP(std::vector<CallInst *> &CallList, + uint32_t NumOfZerosIndex, uint32_t DIIndex); + + Value *computeBaseAndAccessStr(CallInst *Call, std::string &AccessStr, + std::string &AccessKey, uint32_t Kind, + MDNode *&TypeMeta); + bool getAccessIndex(const Value *IndexValue, uint64_t &AccessIndex); + bool transformGEPChain(Module &M, CallInst *Call, uint32_t Kind); +}; +} // End anonymous namespace + +char BPFAbstractMemberAccess::ID = 0; +INITIALIZE_PASS(BPFAbstractMemberAccess, DEBUG_TYPE, + "abstracting struct/union member accessees", false, false) + +ModulePass *llvm::createBPFAbstractMemberAccess() { + return new BPFAbstractMemberAccess(); +} + +bool BPFAbstractMemberAccess::runOnModule(Module &M) { + LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n"); + + // Bail out if no debug info. + if (empty(M.debug_compile_units())) + return false; + + return doTransformation(M); +} + +/// Check whether a call is a preserve_*_access_index intrinsic call or not. +bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call, + uint32_t &Kind) { + if (!Call) + return false; + + const auto *GV = dyn_cast<GlobalValue>(Call->getCalledValue()); + if (!GV) + return false; + if (GV->getName().startswith("llvm.preserve.array.access.index")) { + Kind = BPFPreserveArrayAI; + return true; + } + if (GV->getName().startswith("llvm.preserve.union.access.index")) { + Kind = BPFPreserveUnionAI; + return true; + } + if (GV->getName().startswith("llvm.preserve.struct.access.index")) { + Kind = BPFPreserveStructAI; + return true; + } + + return false; +} + +void BPFAbstractMemberAccess::replaceWithGEP(std::vector<CallInst *> &CallList, + uint32_t DimensionIndex, + uint32_t GEPIndex) { + for (auto Call : CallList) { + uint32_t Dimension = 1; + if (DimensionIndex > 0) + Dimension = cast<ConstantInt>(Call->getArgOperand(DimensionIndex)) + ->getZExtValue(); + + Constant *Zero = + ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0); + SmallVector<Value *, 4> IdxList; + for (unsigned I = 0; I < Dimension; ++I) + IdxList.push_back(Zero); + IdxList.push_back(Call->getArgOperand(GEPIndex)); + + auto *GEP = GetElementPtrInst::CreateInBounds(Call->getArgOperand(0), + IdxList, "", Call); + Call->replaceAllUsesWith(GEP); + Call->eraseFromParent(); + } +} + +bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Module &M) { + std::vector<CallInst *> PreserveArrayIndexCalls; + std::vector<CallInst *> PreserveUnionIndexCalls; + std::vector<CallInst *> PreserveStructIndexCalls; + bool Found = false; + + for (Function &F : M) + for (auto &BB : F) + for (auto &I : BB) { + auto *Call = dyn_cast<CallInst>(&I); + uint32_t Kind; + if (!IsPreserveDIAccessIndexCall(Call, Kind)) + continue; + + Found = true; + if (Kind == BPFPreserveArrayAI) + PreserveArrayIndexCalls.push_back(Call); + else if (Kind == BPFPreserveUnionAI) + PreserveUnionIndexCalls.push_back(Call); + else + PreserveStructIndexCalls.push_back(Call); + } + + // do the following transformation: + // . addr = preserve_array_access_index(base, dimension, index) + // is transformed to + // addr = GEP(base, dimenion's zero's, index) + // . addr = preserve_union_access_index(base, di_index) + // is transformed to + // addr = base, i.e., all usages of "addr" are replaced by "base". + // . addr = preserve_struct_access_index(base, gep_index, di_index) + // is transformed to + // addr = GEP(base, 0, gep_index) + replaceWithGEP(PreserveArrayIndexCalls, 1, 2); + replaceWithGEP(PreserveStructIndexCalls, 0, 1); + for (auto Call : PreserveUnionIndexCalls) { + Call->replaceAllUsesWith(Call->getArgOperand(0)); + Call->eraseFromParent(); + } + + return Found; +} + +void BPFAbstractMemberAccess::traceAICall(CallInst *Call, uint32_t Kind) { + for (User *U : Call->users()) { + Instruction *Inst = dyn_cast<Instruction>(U); + if (!Inst) + continue; + + if (auto *BI = dyn_cast<BitCastInst>(Inst)) { + traceBitCast(BI, Call, Kind); + } else if (auto *CI = dyn_cast<CallInst>(Inst)) { + uint32_t CIKind; + if (IsPreserveDIAccessIndexCall(CI, CIKind)) { + AIChain[CI] = std::make_pair(Call, Kind); + traceAICall(CI, CIKind); + } else { + BaseAICalls[Call] = Kind; + } + } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { + if (GI->hasAllZeroIndices()) + traceGEP(GI, Call, Kind); + else + BaseAICalls[Call] = Kind; + } + } +} + +void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast, + CallInst *Parent, uint32_t Kind) { + for (User *U : BitCast->users()) { + Instruction *Inst = dyn_cast<Instruction>(U); + if (!Inst) + continue; + + if (auto *BI = dyn_cast<BitCastInst>(Inst)) { + traceBitCast(BI, Parent, Kind); + } else if (auto *CI = dyn_cast<CallInst>(Inst)) { + uint32_t CIKind; + if (IsPreserveDIAccessIndexCall(CI, CIKind)) { + AIChain[CI] = std::make_pair(Parent, Kind); + traceAICall(CI, CIKind); + } else { + BaseAICalls[Parent] = Kind; + } + } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { + if (GI->hasAllZeroIndices()) + traceGEP(GI, Parent, Kind); + else + BaseAICalls[Parent] = Kind; + } + } +} + +void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent, + uint32_t Kind) { + for (User *U : GEP->users()) { + Instruction *Inst = dyn_cast<Instruction>(U); + if (!Inst) + continue; + + if (auto *BI = dyn_cast<BitCastInst>(Inst)) { + traceBitCast(BI, Parent, Kind); + } else if (auto *CI = dyn_cast<CallInst>(Inst)) { + uint32_t CIKind; + if (IsPreserveDIAccessIndexCall(CI, CIKind)) { + AIChain[CI] = std::make_pair(Parent, Kind); + traceAICall(CI, CIKind); + } else { + BaseAICalls[Parent] = Kind; + } + } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { + if (GI->hasAllZeroIndices()) + traceGEP(GI, Parent, Kind); + else + BaseAICalls[Parent] = Kind; + } + } +} + +void BPFAbstractMemberAccess::collectAICallChains(Module &M, Function &F) { + AIChain.clear(); + BaseAICalls.clear(); + + for (auto &BB : F) + for (auto &I : BB) { + uint32_t Kind; + auto *Call = dyn_cast<CallInst>(&I); + if (!IsPreserveDIAccessIndexCall(Call, Kind) || + AIChain.find(Call) != AIChain.end()) + continue; + + traceAICall(Call, Kind); + } +} + +/// Get access index from the preserve_*_access_index intrinsic calls. +bool BPFAbstractMemberAccess::getAccessIndex(const Value *IndexValue, + uint64_t &AccessIndex) { + const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue); + if (!CV) + return false; + + AccessIndex = CV->getValue().getZExtValue(); + return true; +} + +/// Compute the base of the whole preserve_*_access_index chains, i.e., the base +/// pointer of the first preserve_*_access_index call, and construct the access +/// string, which will be the name of a global variable. +Value *BPFAbstractMemberAccess::computeBaseAndAccessStr(CallInst *Call, + std::string &AccessStr, + std::string &AccessKey, + uint32_t Kind, + MDNode *&TypeMeta) { + Value *Base = nullptr; + std::vector<uint64_t> AccessIndices; + uint64_t TypeNameIndex = 0; + std::string LastTypeName; + + while (Call) { + // Base of original corresponding GEP + Base = Call->getArgOperand(0); + + // Type Name + std::string TypeName; + MDNode *MDN; + if (Kind == BPFPreserveUnionAI || Kind == BPFPreserveStructAI) { + MDN = Call->getMetadata(LLVMContext::MD_preserve_access_index); + if (!MDN) + return nullptr; + + DIType *Ty = dyn_cast<DIType>(MDN); + if (!Ty) + return nullptr; + + TypeName = Ty->getName(); + } + + // Access Index + uint64_t AccessIndex; + uint32_t ArgIndex = (Kind == BPFPreserveUnionAI) ? 1 : 2; + if (!getAccessIndex(Call->getArgOperand(ArgIndex), AccessIndex)) + return nullptr; + + AccessIndices.push_back(AccessIndex); + if (TypeName.size()) { + TypeNameIndex = AccessIndices.size() - 1; + LastTypeName = TypeName; + TypeMeta = MDN; + } + + Kind = AIChain[Call].second; + Call = AIChain[Call].first; + } + + // The intial type name is required. + // FIXME: if the initial type access is an array index, e.g., + // &a[3].b.c, only one dimentional array is supported. + if (!LastTypeName.size() || AccessIndices.size() > TypeNameIndex + 2) + return nullptr; + + // Construct the type string AccessStr. + for (unsigned I = 0; I < AccessIndices.size(); ++I) + AccessStr = std::to_string(AccessIndices[I]) + ":" + AccessStr; + + if (TypeNameIndex == AccessIndices.size() - 1) + AccessStr = "0:" + AccessStr; + + // Access key is the type name + access string, uniquely identifying + // one kernel memory access. + AccessKey = LastTypeName + ":" + AccessStr; + + return Base; +} + +/// Call/Kind is the base preserve_*_access_index() call. Attempts to do +/// transformation to a chain of relocable GEPs. +bool BPFAbstractMemberAccess::transformGEPChain(Module &M, CallInst *Call, + uint32_t Kind) { + std::string AccessStr, AccessKey; + MDNode *TypeMeta = nullptr; + Value *Base = + computeBaseAndAccessStr(Call, AccessStr, AccessKey, Kind, TypeMeta); + if (!Base) + return false; + + // Do the transformation + // For any original GEP Call and Base %2 like + // %4 = bitcast %struct.net_device** %dev1 to i64* + // it is transformed to: + // %6 = load __BTF_0:sk_buff:0:0:2:0: + // %7 = bitcast %struct.sk_buff* %2 to i8* + // %8 = getelementptr i8, i8* %7, %6 + // %9 = bitcast i8* %8 to i64* + // using %9 instead of %4 + // The original Call inst is removed. + BasicBlock *BB = Call->getParent(); + GlobalVariable *GV; + + if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) { + GV = new GlobalVariable(M, Type::getInt64Ty(BB->getContext()), false, + GlobalVariable::ExternalLinkage, NULL, AccessStr); + GV->addAttribute(BPFCoreSharedInfo::AmaAttr); + // Set the metadata (debuginfo types) for the global. + if (TypeMeta) + GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); + GEPGlobals[AccessKey] = GV; + } else { + GV = GEPGlobals[AccessKey]; + } + + // Load the global variable. + auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV); + BB->getInstList().insert(Call->getIterator(), LDInst); + + // Generate a BitCast + auto *BCInst = new BitCastInst(Base, Type::getInt8PtrTy(BB->getContext())); + BB->getInstList().insert(Call->getIterator(), BCInst); + + // Generate a GetElementPtr + auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()), + BCInst, LDInst); + BB->getInstList().insert(Call->getIterator(), GEP); + + // Generate a BitCast + auto *BCInst2 = new BitCastInst(GEP, Call->getType()); + BB->getInstList().insert(Call->getIterator(), BCInst2); + + Call->replaceAllUsesWith(BCInst2); + Call->eraseFromParent(); + + return true; +} + +bool BPFAbstractMemberAccess::doTransformation(Module &M) { + bool Transformed = false; + + for (Function &F : M) { + // Collect PreserveDIAccessIndex Intrinsic call chains. + // The call chains will be used to generate the access + // patterns similar to GEP. + collectAICallChains(M, F); + + for (auto &C : BaseAICalls) + Transformed = transformGEPChain(M, C.first, C.second) || Transformed; + } + + return removePreserveAccessIndexIntrinsic(M) || Transformed; +} diff --git a/contrib/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/contrib/llvm/lib/Target/BPF/BPFAsmPrinter.cpp index ada5eb923f40..e61e73468057 100644 --- a/contrib/llvm/lib/Target/BPF/BPFAsmPrinter.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFAsmPrinter.cpp @@ -1,9 +1,8 @@ //===-- BPFAsmPrinter.cpp - BPF LLVM assembly writer ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,8 @@ #include "BPFMCInstLower.h" #include "BPFTargetMachine.h" #include "BTFDebug.h" -#include "InstPrinter/BPFInstPrinter.h" +#include "MCTargetDesc/BPFInstPrinter.h" +#include "TargetInfo/BPFTargetInfo.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -38,27 +38,30 @@ class BPFAsmPrinter : public AsmPrinter { public: explicit BPFAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) - : AsmPrinter(TM, std::move(Streamer)) {} + : AsmPrinter(TM, std::move(Streamer)), BTF(nullptr) {} StringRef getPassName() const override { return "BPF Assembly Printer"; } bool doInitialization(Module &M) override; void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O); bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &O) override; + const char *ExtraCode, raw_ostream &O) override; bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &O) override; + const char *ExtraCode, raw_ostream &O) override; void EmitInstruction(const MachineInstr *MI) override; + +private: + BTFDebug *BTF; }; } // namespace bool BPFAsmPrinter::doInitialization(Module &M) { AsmPrinter::doInitialization(M); - if (MAI->doesSupportDebugInformation()) { - Handlers.push_back(HandlerInfo(new BTFDebug(this), "emit", + // Only emit BTF when debuginfo available. + if (MAI->doesSupportDebugInformation() && !empty(M.debug_compile_units())) { + BTF = new BTFDebug(this); + Handlers.push_back(HandlerInfo(std::unique_ptr<BTFDebug>(BTF), "emit", "Debug Info Emission", "BTF", "BTF Emission")); } @@ -105,18 +108,16 @@ void BPFAsmPrinter::printOperand(const MachineInstr *MI, int OpNum, } bool BPFAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - unsigned /*AsmVariant*/, const char *ExtraCode, raw_ostream &O) { if (ExtraCode && ExtraCode[0]) - return true; // BPF does not have special modifiers + return AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, O); printOperand(MI, OpNo, O); return false; } bool BPFAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, - unsigned OpNum, unsigned AsmVariant, - const char *ExtraCode, + unsigned OpNum, const char *ExtraCode, raw_ostream &O) { assert(OpNum + 1 < MI->getNumOperands() && "Insufficient operands"); const MachineOperand &BaseMO = MI->getOperand(OpNum); @@ -137,11 +138,12 @@ bool BPFAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, } void BPFAsmPrinter::EmitInstruction(const MachineInstr *MI) { - - BPFMCInstLower MCInstLowering(OutContext, *this); - MCInst TmpInst; - MCInstLowering.Lower(MI, TmpInst); + + if (!BTF || !BTF->InstLower(MI, TmpInst)) { + BPFMCInstLower MCInstLowering(OutContext, *this); + MCInstLowering.Lower(MI, TmpInst); + } EmitToStreamer(*OutStreamer, TmpInst); } diff --git a/contrib/llvm/lib/Target/BPF/BPFCORE.h b/contrib/llvm/lib/Target/BPF/BPFCORE.h new file mode 100644 index 000000000000..e0950d95f8d7 --- /dev/null +++ b/contrib/llvm/lib/Target/BPF/BPFCORE.h @@ -0,0 +1,24 @@ +//===- BPFCORE.h - Common info for Compile-Once Run-EveryWhere -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_BPFCORE_H +#define LLVM_LIB_TARGET_BPF_BPFCORE_H + +namespace llvm { + +class BPFCoreSharedInfo { +public: + /// The attribute attached to globals representing a member offset + static const std::string AmaAttr; + /// The section name to identify a patchable external global + static const std::string PatchableExtSecName; +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm/lib/Target/BPF/BPFCallingConv.td b/contrib/llvm/lib/Target/BPF/BPFCallingConv.td index 637f9752ec42..ef4ef1930aa8 100644 --- a/contrib/llvm/lib/Target/BPF/BPFCallingConv.td +++ b/contrib/llvm/lib/Target/BPF/BPFCallingConv.td @@ -1,9 +1,8 @@ //===-- BPFCallingConv.td - Calling Conventions BPF --------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFFrameLowering.cpp b/contrib/llvm/lib/Target/BPF/BPFFrameLowering.cpp index c2806c85f24f..8812cfdd86da 100644 --- a/contrib/llvm/lib/Target/BPF/BPFFrameLowering.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFFrameLowering.cpp @@ -1,9 +1,8 @@ //===-- BPFFrameLowering.cpp - BPF Frame Information ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFFrameLowering.h b/contrib/llvm/lib/Target/BPF/BPFFrameLowering.h index b4ffa0713fa6..2dc6277d2244 100644 --- a/contrib/llvm/lib/Target/BPF/BPFFrameLowering.h +++ b/contrib/llvm/lib/Target/BPF/BPFFrameLowering.h @@ -1,9 +1,8 @@ //===-- BPFFrameLowering.h - Define frame lowering for BPF -----*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp b/contrib/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp index 8b9bc08e144f..1bd705c55188 100644 --- a/contrib/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp @@ -1,9 +1,8 @@ //===-- BPFISelDAGToDAG.cpp - A dag to dag inst selector for BPF ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFISelLowering.cpp b/contrib/llvm/lib/Target/BPF/BPFISelLowering.cpp index 9272cf692dc9..ff69941d26fb 100644 --- a/contrib/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -1,9 +1,8 @@ //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -106,7 +105,8 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, if (STI.getHasAlu32()) { setOperationAction(ISD::BSWAP, MVT::i32, Promote); - setOperationAction(ISD::BR_CC, MVT::i32, Promote); + setOperationAction(ISD::BR_CC, MVT::i32, + STI.getHasJmp32() ? Custom : Promote); } setOperationAction(ISD::CTTZ, MVT::i64, Custom); @@ -163,6 +163,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, // CPU/Feature control HasAlu32 = STI.getHasAlu32(); + HasJmp32 = STI.getHasJmp32(); HasJmpExt = STI.getHasJmpExt(); } @@ -507,7 +508,7 @@ SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { NegateCC(LHS, RHS, CC); return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS, - DAG.getConstant(CC, DL, MVT::i64), Dest); + DAG.getConstant(CC, DL, LHS.getValueType()), Dest); } SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { @@ -677,36 +678,23 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, int CC = MI.getOperand(3).getImm(); int NewCC; switch (CC) { - case ISD::SETGT: - NewCC = isSelectRROp ? BPF::JSGT_rr : BPF::JSGT_ri; - break; - case ISD::SETUGT: - NewCC = isSelectRROp ? BPF::JUGT_rr : BPF::JUGT_ri; - break; - case ISD::SETGE: - NewCC = isSelectRROp ? BPF::JSGE_rr : BPF::JSGE_ri; - break; - case ISD::SETUGE: - NewCC = isSelectRROp ? BPF::JUGE_rr : BPF::JUGE_ri; - break; - case ISD::SETEQ: - NewCC = isSelectRROp ? BPF::JEQ_rr : BPF::JEQ_ri; - break; - case ISD::SETNE: - NewCC = isSelectRROp ? BPF::JNE_rr : BPF::JNE_ri; - break; - case ISD::SETLT: - NewCC = isSelectRROp ? BPF::JSLT_rr : BPF::JSLT_ri; - break; - case ISD::SETULT: - NewCC = isSelectRROp ? BPF::JULT_rr : BPF::JULT_ri; - break; - case ISD::SETLE: - NewCC = isSelectRROp ? BPF::JSLE_rr : BPF::JSLE_ri; - break; - case ISD::SETULE: - NewCC = isSelectRROp ? BPF::JULE_rr : BPF::JULE_ri; - break; +#define SET_NEWCC(X, Y) \ + case ISD::X: \ + if (is32BitCmp && HasJmp32) \ + NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \ + else \ + NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \ + break + SET_NEWCC(SETGT, JSGT); + SET_NEWCC(SETUGT, JUGT); + SET_NEWCC(SETGE, JSGE); + SET_NEWCC(SETUGE, JUGE); + SET_NEWCC(SETEQ, JEQ); + SET_NEWCC(SETNE, JNE); + SET_NEWCC(SETLT, JSLT); + SET_NEWCC(SETULT, JULT); + SET_NEWCC(SETLE, JSLE); + SET_NEWCC(SETULE, JULE); default: report_fatal_error("unimplemented select CondCode " + Twine(CC)); } @@ -724,13 +712,13 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, // // We simply do extension for all situations in this method, but we will // try to remove those unnecessary in BPFMIPeephole pass. - if (is32BitCmp) + if (is32BitCmp && !HasJmp32) LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp); if (isSelectRROp) { unsigned RHS = MI.getOperand(2).getReg(); - if (is32BitCmp) + if (is32BitCmp && !HasJmp32) RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp); BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB); diff --git a/contrib/llvm/lib/Target/BPF/BPFISelLowering.h b/contrib/llvm/lib/Target/BPF/BPFISelLowering.h index 0aa8b9ac57ac..b81bf4e1320d 100644 --- a/contrib/llvm/lib/Target/BPF/BPFISelLowering.h +++ b/contrib/llvm/lib/Target/BPF/BPFISelLowering.h @@ -1,9 +1,8 @@ //===-- BPFISelLowering.h - BPF DAG Lowering Interface ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,6 +55,7 @@ public: MachineBasicBlock *BB) const override; bool getHasAlu32() const { return HasAlu32; } + bool getHasJmp32() const { return HasJmp32; } bool getHasJmpExt() const { return HasJmpExt; } EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, @@ -66,6 +66,7 @@ public: private: // Control Instruction Selection Features bool HasAlu32; + bool HasJmp32; bool HasJmpExt; SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const; @@ -100,7 +101,7 @@ private: EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, - MachineFunction &MF) const override { + const AttributeList &FuncAttributes) const override { return Size >= 8 ? MVT::i64 : MVT::i32; } diff --git a/contrib/llvm/lib/Target/BPF/BPFInstrFormats.td b/contrib/llvm/lib/Target/BPF/BPFInstrFormats.td index 92d4a62fd875..9f00dc85d789 100644 --- a/contrib/llvm/lib/Target/BPF/BPFInstrFormats.td +++ b/contrib/llvm/lib/Target/BPF/BPFInstrFormats.td @@ -1,9 +1,8 @@ //===-- BPFInstrFormats.td - BPF Instruction Formats -------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -17,6 +16,7 @@ def BPF_ST : BPFOpClass<0x2>; def BPF_STX : BPFOpClass<0x3>; def BPF_ALU : BPFOpClass<0x4>; def BPF_JMP : BPFOpClass<0x5>; +def BPF_JMP32 : BPFOpClass<0x6>; def BPF_ALU64 : BPFOpClass<0x7>; class BPFSrcType<bits<1> val> { diff --git a/contrib/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/contrib/llvm/lib/Target/BPF/BPFInstrInfo.cpp index 4d47debdaa74..932f718d5490 100644 --- a/contrib/llvm/lib/Target/BPF/BPFInstrInfo.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFInstrInfo.cpp @@ -1,9 +1,8 @@ //===-- BPFInstrInfo.cpp - BPF Instruction Information ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFInstrInfo.h b/contrib/llvm/lib/Target/BPF/BPFInstrInfo.h index fb65a86a6d18..e4bd757da560 100644 --- a/contrib/llvm/lib/Target/BPF/BPFInstrInfo.h +++ b/contrib/llvm/lib/Target/BPF/BPFInstrInfo.h @@ -1,9 +1,8 @@ //===-- BPFInstrInfo.h - BPF Instruction Information ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFInstrInfo.td b/contrib/llvm/lib/Target/BPF/BPFInstrInfo.td index aaef5fb706e0..c44702a78ec8 100644 --- a/contrib/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/contrib/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -1,9 +1,8 @@ //===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -102,6 +101,26 @@ def BPF_CC_LTU : PatLeaf<(i64 imm), [{return (N->getZExtValue() == ISD::SETULT);}]>; def BPF_CC_LEU : PatLeaf<(i64 imm), [{return (N->getZExtValue() == ISD::SETULE);}]>; +def BPF_CC_EQ_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETEQ);}]>; +def BPF_CC_NE_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETNE);}]>; +def BPF_CC_GE_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETGE);}]>; +def BPF_CC_GT_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETGT);}]>; +def BPF_CC_GTU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETUGT);}]>; +def BPF_CC_GEU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETUGE);}]>; +def BPF_CC_LE_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETLE);}]>; +def BPF_CC_LT_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETLT);}]>; +def BPF_CC_LTU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETULT);}]>; +def BPF_CC_LEU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETULE);}]>; // For arithmetic and jump instructions the 8-bit 'code' // field is divided into three parts: @@ -167,23 +186,57 @@ class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> let BPFClass = BPF_JMP; } -multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> { +class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> + : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, + (outs), + (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst), + "if $dst "#OpcodeStr#" $src goto $BrDst", + [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> { + bits<4> dst; + bits<4> src; + bits<16> BrDst; + + let Inst{55-52} = src; + let Inst{51-48} = dst; + let Inst{47-32} = BrDst; + let BPFClass = BPF_JMP32; +} + +class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> + : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, + (outs), + (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst), + "if $dst "#OpcodeStr#" $imm goto $BrDst", + [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> { + bits<4> dst; + bits<16> BrDst; + bits<32> imm; + + let Inst{51-48} = dst; + let Inst{47-32} = BrDst; + let Inst{31-0} = imm; + let BPFClass = BPF_JMP32; +} + +multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> { def _rr : JMP_RR<Opc, OpcodeStr, Cond>; def _ri : JMP_RI<Opc, OpcodeStr, Cond>; + def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>; + def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>; } let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in { // cmp+goto instructions -defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ>; -defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU>; -defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU>; -defm JNE : J<BPF_JNE, "!=", BPF_CC_NE>; -defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT>; -defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE>; -defm JULT : J<BPF_JLT, "<", BPF_CC_LTU>; -defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU>; -defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT>; -defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE>; +defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>; +defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>; +defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>; +defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>; +defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>; +defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>; +defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>; +defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>; +defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>; +defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>; } // ALU instructions @@ -561,11 +614,31 @@ class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> let BPFClass = BPF_STX; } +class XADD32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> + : TYPE_LD_ST<BPF_XADD.Value, SizeOp.Value, + (outs GPR32:$dst), + (ins MEMri:$addr, GPR32:$val), + "lock *("#OpcodeStr#" *)($addr) += $val", + [(set GPR32:$dst, (OpNode ADDRri:$addr, GPR32:$val))]> { + bits<4> dst; + bits<20> addr; + + let Inst{51-48} = addr{19-16}; // base reg + let Inst{55-52} = dst; + let Inst{47-32} = addr{15-0}; // offset + let BPFClass = BPF_STX; +} + let Constraints = "$dst = $val" in { -def XADD32 : XADD<BPF_W, "u32", atomic_load_add_32>; -def XADD64 : XADD<BPF_DW, "u64", atomic_load_add_64>; -// undefined def XADD16 : XADD<1, "xadd16", atomic_load_add_16>; -// undefined def XADD8 : XADD<2, "xadd8", atomic_load_add_8>; + let Predicates = [BPFNoALU32] in { + def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>; + } + + let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { + def XADDW32 : XADD32<BPF_W, "u32", atomic_load_add_32>; + } + + def XADDD : XADD<BPF_DW, "u64", atomic_load_add_64>; } // bswap16, bswap32, bswap64 diff --git a/contrib/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/contrib/llvm/lib/Target/BPF/BPFMCInstLower.cpp index c8528e867310..846798a63cb7 100644 --- a/contrib/llvm/lib/Target/BPF/BPFMCInstLower.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFMCInstLower.cpp @@ -1,9 +1,8 @@ //=-- BPFMCInstLower.cpp - Convert BPF MachineInstr to an MCInst ------------=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFMCInstLower.h b/contrib/llvm/lib/Target/BPF/BPFMCInstLower.h index eac811f4cf88..0622d20814d3 100644 --- a/contrib/llvm/lib/Target/BPF/BPFMCInstLower.h +++ b/contrib/llvm/lib/Target/BPF/BPFMCInstLower.h @@ -1,9 +1,8 @@ //===-- BPFMCInstLower.h - Lower MachineInstr to MCInst ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/Target/BPF/BPFMIChecking.cpp b/contrib/llvm/lib/Target/BPF/BPFMIChecking.cpp index 0a311378e777..4c46289656b4 100644 --- a/contrib/llvm/lib/Target/BPF/BPFMIChecking.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFMIChecking.cpp @@ -1,9 +1,8 @@ //===-------------- BPFMIChecking.cpp - MI Checking Legality -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,14 +61,107 @@ void BPFMIPreEmitChecking::initialize(MachineFunction &MFParm) { LLVM_DEBUG(dbgs() << "*** BPF PreEmit checking pass ***\n\n"); } +// Make sure all Defs of XADD are dead, meaning any result of XADD insn is not +// used. +// +// NOTE: BPF backend hasn't enabled sub-register liveness track, so when the +// source and destination operands of XADD are GPR32, there is no sub-register +// dead info. If we rely on the generic MachineInstr::allDefsAreDead, then we +// will raise false alarm on GPR32 Def. +// +// To support GPR32 Def, ideally we could just enable sub-registr liveness track +// on BPF backend, then allDefsAreDead could work on GPR32 Def. This requires +// implementing TargetSubtargetInfo::enableSubRegLiveness on BPF. +// +// However, sub-register liveness tracking module inside LLVM is actually +// designed for the situation where one register could be split into more than +// one sub-registers for which case each sub-register could have their own +// liveness and kill one of them doesn't kill others. So, tracking liveness for +// each make sense. +// +// For BPF, each 64-bit register could only have one 32-bit sub-register. This +// is exactly the case which LLVM think brings no benefits for doing +// sub-register tracking, because the live range of sub-register must always +// equal to its parent register, therefore liveness tracking is disabled even +// the back-end has implemented enableSubRegLiveness. The detailed information +// is at r232695: +// +// Author: Matthias Braun <matze@braunis.de> +// Date: Thu Mar 19 00:21:58 2015 +0000 +// Do not track subregister liveness when it brings no benefits +// +// Hence, for BPF, we enhance MachineInstr::allDefsAreDead. Given the solo +// sub-register always has the same liveness as its parent register, LLVM is +// already attaching a implicit 64-bit register Def whenever the there is +// a sub-register Def. The liveness of the implicit 64-bit Def is available. +// For example, for "lock *(u32 *)(r0 + 4) += w9", the MachineOperand info could +// be: +// +// $w9 = XADDW32 killed $r0, 4, $w9(tied-def 0), +// implicit killed $r9, implicit-def dead $r9 +// +// Even though w9 is not marked as Dead, the parent register r9 is marked as +// Dead correctly, and it is safe to use such information or our purpose. +static bool hasLiveDefs(const MachineInstr &MI, const TargetRegisterInfo *TRI) { + const MCRegisterClass *GPR64RegClass = + &BPFMCRegisterClasses[BPF::GPRRegClassID]; + std::vector<unsigned> GPR32LiveDefs; + std::vector<unsigned> GPR64DeadDefs; + + for (const MachineOperand &MO : MI.operands()) { + bool RegIsGPR64; + + if (!MO.isReg() || MO.isUse()) + continue; + + RegIsGPR64 = GPR64RegClass->contains(MO.getReg()); + if (!MO.isDead()) { + // It is a GPR64 live Def, we are sure it is live. */ + if (RegIsGPR64) + return true; + // It is a GPR32 live Def, we are unsure whether it is really dead due to + // no sub-register liveness tracking. Push it to vector for deferred + // check. + GPR32LiveDefs.push_back(MO.getReg()); + continue; + } + + // Record any GPR64 dead Def as some unmarked GPR32 could be alias of its + // low 32-bit. + if (RegIsGPR64) + GPR64DeadDefs.push_back(MO.getReg()); + } + + // No GPR32 live Def, safe to return false. + if (GPR32LiveDefs.empty()) + return false; + + // No GPR64 dead Def, so all those GPR32 live Def can't have alias, therefore + // must be truely live, safe to return true. + if (GPR64DeadDefs.empty()) + return true; + + // Otherwise, return true if any aliased SuperReg of GPR32 is not dead. + std::vector<unsigned>::iterator search_begin = GPR64DeadDefs.begin(); + std::vector<unsigned>::iterator search_end = GPR64DeadDefs.end(); + for (auto I : GPR32LiveDefs) + for (MCSuperRegIterator SR(I, TRI); SR.isValid(); ++SR) + if (std::find(search_begin, search_end, *SR) == search_end) + return true; + + return false; +} + void BPFMIPreEmitChecking::checkingIllegalXADD(void) { for (MachineBasicBlock &MBB : *MF) { for (MachineInstr &MI : MBB) { - if (MI.getOpcode() != BPF::XADD32 && MI.getOpcode() != BPF::XADD64) + if (MI.getOpcode() != BPF::XADDW && + MI.getOpcode() != BPF::XADDD && + MI.getOpcode() != BPF::XADDW32) continue; LLVM_DEBUG(MI.dump()); - if (!MI.allDefsAreDead()) { + if (hasLiveDefs(MI, TRI)) { DebugLoc Empty; const DebugLoc &DL = MI.getDebugLoc(); if (DL != Empty) diff --git a/contrib/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/contrib/llvm/lib/Target/BPF/BPFMIPeephole.cpp index 9e984d0facfb..156ba793e359 100644 --- a/contrib/llvm/lib/Target/BPF/BPFMIPeephole.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFMIPeephole.cpp @@ -1,9 +1,8 @@ //===-------------- BPFMIPeephole.cpp - MI Peephole Cleanups -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp b/contrib/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp new file mode 100644 index 000000000000..e9114d7187e3 --- /dev/null +++ b/contrib/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp @@ -0,0 +1,163 @@ +//===----- BPFMISimplifyPatchable.cpp - MI Simplify Patchable Insts -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass targets a subset of instructions like below +// ld_imm64 r1, @global +// ldd r2, r1, 0 +// add r3, struct_base_reg, r2 +// +// Here @global should either present a AMA (abstruct member access) or +// a patchable extern variable. And these two kinds of accesses +// are subject to bpf load time patching. After this pass, the +// code becomes +// ld_imm64 r1, @global +// add r3, struct_base_reg, r1 +// +// Eventually, at BTF output stage, a relocation record will be generated +// for ld_imm64 which should be replaced later by bpf loader: +// r1 = <calculated offset> or <to_be_patched_extern_val> +// add r3, struct_base_reg, r1 +// or +// ld_imm64 r1, <to_be_patched_extern_val> +// add r3, struct_base_reg, r1 +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include "BPFCORE.h" +#include "BPFInstrInfo.h" +#include "BPFTargetMachine.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" + +using namespace llvm; + +#define DEBUG_TYPE "bpf-mi-simplify-patchable" + +namespace { + +struct BPFMISimplifyPatchable : public MachineFunctionPass { + + static char ID; + const BPFInstrInfo *TII; + MachineFunction *MF; + + BPFMISimplifyPatchable() : MachineFunctionPass(ID) { + initializeBPFMISimplifyPatchablePass(*PassRegistry::getPassRegistry()); + } + +private: + // Initialize class variables. + void initialize(MachineFunction &MFParm); + + bool removeLD(void); + +public: + // Main entry point for this pass. + bool runOnMachineFunction(MachineFunction &MF) override { + if (!skipFunction(MF.getFunction())) { + initialize(MF); + } + return removeLD(); + } +}; + +// Initialize class variables. +void BPFMISimplifyPatchable::initialize(MachineFunction &MFParm) { + MF = &MFParm; + TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo(); + LLVM_DEBUG(dbgs() << "*** BPF simplify patchable insts pass ***\n\n"); +} + +/// Remove unneeded Load instructions. +bool BPFMISimplifyPatchable::removeLD() { + MachineRegisterInfo *MRI = &MF->getRegInfo(); + MachineInstr *ToErase = nullptr; + bool Changed = false; + + for (MachineBasicBlock &MBB : *MF) { + for (MachineInstr &MI : MBB) { + if (ToErase) { + ToErase->eraseFromParent(); + ToErase = nullptr; + } + + // Ensure the register format is LOAD <reg>, <reg>, 0 + if (MI.getOpcode() != BPF::LDD && MI.getOpcode() != BPF::LDW && + MI.getOpcode() != BPF::LDH && MI.getOpcode() != BPF::LDB && + MI.getOpcode() != BPF::LDW32 && MI.getOpcode() != BPF::LDH32 && + MI.getOpcode() != BPF::LDB32) + continue; + + if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg()) + continue; + + if (!MI.getOperand(2).isImm() || MI.getOperand(2).getImm()) + continue; + + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = MI.getOperand(1).getReg(); + int64_t ImmVal = MI.getOperand(2).getImm(); + + MachineInstr *DefInst = MRI->getUniqueVRegDef(SrcReg); + if (!DefInst) + continue; + + bool IsCandidate = false; + if (DefInst->getOpcode() == BPF::LD_imm64) { + const MachineOperand &MO = DefInst->getOperand(1); + if (MO.isGlobal()) { + const GlobalValue *GVal = MO.getGlobal(); + auto *GVar = dyn_cast<GlobalVariable>(GVal); + if (GVar) { + // Global variables representing structure offset or + // patchable extern globals. + if (GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) { + assert(ImmVal == 0); + IsCandidate = true; + } else if (!GVar->hasInitializer() && GVar->hasExternalLinkage() && + GVar->getSection() == + BPFCoreSharedInfo::PatchableExtSecName) { + if (ImmVal == 0) + IsCandidate = true; + else + errs() << "WARNING: unhandled patchable extern " + << GVar->getName() << " with load offset " << ImmVal + << "\n"; + } + } + } + } + + if (!IsCandidate) + continue; + + auto Begin = MRI->use_begin(DstReg), End = MRI->use_end(); + decltype(End) NextI; + for (auto I = Begin; I != End; I = NextI) { + NextI = std::next(I); + I->setReg(SrcReg); + } + + ToErase = &MI; + Changed = true; + } + } + + return Changed; +} + +} // namespace + +INITIALIZE_PASS(BPFMISimplifyPatchable, DEBUG_TYPE, + "BPF PreEmit SimplifyPatchable", false, false) + +char BPFMISimplifyPatchable::ID = 0; +FunctionPass *llvm::createBPFMISimplifyPatchablePass() { + return new BPFMISimplifyPatchable(); +} diff --git a/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.cpp b/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.cpp index 635c11113151..714af06e11d9 100644 --- a/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.cpp @@ -1,9 +1,8 @@ //===-- BPFRegisterInfo.cpp - BPF Register Information ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -122,6 +121,6 @@ void BPFRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, } } -unsigned BPFRegisterInfo::getFrameRegister(const MachineFunction &MF) const { +Register BPFRegisterInfo::getFrameRegister(const MachineFunction &MF) const { return BPF::R10; } diff --git a/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.h b/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.h index 4202850e9eb9..e7b870b720a4 100644 --- a/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.h +++ b/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.h @@ -1,9 +1,8 @@ //===-- BPFRegisterInfo.h - BPF Register Information Impl -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,7 +32,7 @@ struct BPFRegisterInfo : public BPFGenRegisterInfo { unsigned FIOperandNum, RegScavenger *RS = nullptr) const override; - unsigned getFrameRegister(const MachineFunction &MF) const override; + Register getFrameRegister(const MachineFunction &MF) const override; }; } diff --git a/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.td b/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.td index da1d6b505f84..88dec063be70 100644 --- a/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.td +++ b/contrib/llvm/lib/Target/BPF/BPFRegisterInfo.td @@ -1,9 +1,8 @@ //===-- BPFRegisterInfo.td - BPF Register defs -------------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp b/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp index 24d5f59bbfd7..a711294048ba 100644 --- a/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.cpp @@ -1,9 +1,8 @@ //===-- BPFSelectionDAGInfo.cpp - BPF SelectionDAG Info -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h b/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h index 19d3c5769573..fb88c32ceb0c 100644 --- a/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h +++ b/contrib/llvm/lib/Target/BPF/BPFSelectionDAGInfo.h @@ -1,9 +1,8 @@ //===-- BPFSelectionDAGInfo.h - BPF SelectionDAG Info -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BPFSubtarget.cpp b/contrib/llvm/lib/Target/BPF/BPFSubtarget.cpp index 56780bd9d46f..ab3452501b95 100644 --- a/contrib/llvm/lib/Target/BPF/BPFSubtarget.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFSubtarget.cpp @@ -1,9 +1,8 @@ //===-- BPFSubtarget.cpp - BPF Subtarget Information ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,6 +35,7 @@ BPFSubtarget &BPFSubtarget::initializeSubtargetDependencies(StringRef CPU, void BPFSubtarget::initializeEnvironment() { HasJmpExt = false; + HasJmp32 = false; HasAlu32 = false; UseDwarfRIS = false; } @@ -49,6 +49,11 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { HasJmpExt = true; return; } + if (CPU == "v3") { + HasJmpExt = true; + HasJmp32 = true; + return; + } } BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU, diff --git a/contrib/llvm/lib/Target/BPF/BPFSubtarget.h b/contrib/llvm/lib/Target/BPF/BPFSubtarget.h index 60e56435fe4c..3da6a026ab7e 100644 --- a/contrib/llvm/lib/Target/BPF/BPFSubtarget.h +++ b/contrib/llvm/lib/Target/BPF/BPFSubtarget.h @@ -1,9 +1,8 @@ //===-- BPFSubtarget.h - Define Subtarget for the BPF -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -48,6 +47,10 @@ protected: // whether the cpu supports jmp ext bool HasJmpExt; + // whether the cpu supports jmp32 ext. + // NOTE: jmp32 is not enabled when alu32 enabled. + bool HasJmp32; + // whether the cpu supports alu32 instructions. bool HasAlu32; @@ -66,6 +69,7 @@ public: // subtarget options. Definition of function is auto generated by tblgen. void ParseSubtargetFeatures(StringRef CPU, StringRef FS); bool getHasJmpExt() const { return HasJmpExt; } + bool getHasJmp32() const { return HasJmp32; } bool getHasAlu32() const { return HasAlu32; } bool getUseDwarfRIS() const { return UseDwarfRIS; } diff --git a/contrib/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/contrib/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 350465b118ed..24c0ff0f7f15 100644 --- a/contrib/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/contrib/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -1,9 +1,8 @@ //===-- BPFTargetMachine.cpp - Define TargetMachine for BPF ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "BPFTargetMachine.h" #include "BPF.h" #include "MCTargetDesc/BPFMCAsmInfo.h" +#include "TargetInfo/BPFTargetInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/CodeGen/TargetPassConfig.h" @@ -34,6 +34,7 @@ extern "C" void LLVMInitializeBPFTarget() { RegisterTargetMachine<BPFTargetMachine> Z(getTheBPFTarget()); PassRegistry &PR = *PassRegistry::getPassRegistry(); + initializeBPFAbstractMemberAccessPass(PR); initializeBPFMIPeepholePass(PR); } @@ -68,6 +69,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT, static_cast<BPFMCAsmInfo *>(const_cast<MCAsmInfo *>(AsmInfo.get())); MAI->setDwarfUsesRelocationsAcrossSections(!Subtarget.getUseDwarfRIS()); } + namespace { // BPF Code Generator Pass Configuration Options. class BPFPassConfig : public TargetPassConfig { @@ -79,6 +81,7 @@ public: return getTM<BPFTargetMachine>(); } + void addIRPasses() override; bool addInstSelector() override; void addMachineSSAOptimization() override; void addPreEmitPass() override; @@ -89,6 +92,13 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) { return new BPFPassConfig(*this, PM); } +void BPFPassConfig::addIRPasses() { + + addPass(createBPFAbstractMemberAccess()); + + TargetPassConfig::addIRPasses(); +} + // Install an instruction selector pass using // the ISelDag to gen BPF code. bool BPFPassConfig::addInstSelector() { @@ -98,6 +108,8 @@ bool BPFPassConfig::addInstSelector() { } void BPFPassConfig::addMachineSSAOptimization() { + addPass(createBPFMISimplifyPatchablePass()); + // The default implementation must be called first as we want eBPF // Peephole ran at last. TargetPassConfig::addMachineSSAOptimization(); diff --git a/contrib/llvm/lib/Target/BPF/BPFTargetMachine.h b/contrib/llvm/lib/Target/BPF/BPFTargetMachine.h index a560dd27335a..beac7bd862da 100644 --- a/contrib/llvm/lib/Target/BPF/BPFTargetMachine.h +++ b/contrib/llvm/lib/Target/BPF/BPFTargetMachine.h @@ -1,9 +1,8 @@ //===-- BPFTargetMachine.h - Define TargetMachine for BPF --- C++ ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/BTF.def b/contrib/llvm/lib/Target/BPF/BTF.def index 54c5bc3cf092..2d2e9a04aa6d 100644 --- a/contrib/llvm/lib/Target/BPF/BTF.def +++ b/contrib/llvm/lib/Target/BPF/BTF.def @@ -1,9 +1,8 @@ //===- BTF.def - BTF definitions --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,5 +28,7 @@ HANDLE_BTF_KIND(10, CONST) HANDLE_BTF_KIND(11, RESTRICT) HANDLE_BTF_KIND(12, FUNC) HANDLE_BTF_KIND(13, FUNC_PROTO) +HANDLE_BTF_KIND(14, VAR) +HANDLE_BTF_KIND(15, DATASEC) #undef HANDLE_BTF_KIND diff --git a/contrib/llvm/lib/Target/BPF/BTF.h b/contrib/llvm/lib/Target/BPF/BTF.h index 1e1680faf1b8..ad56716710a6 100644 --- a/contrib/llvm/lib/Target/BPF/BTF.h +++ b/contrib/llvm/lib/Target/BPF/BTF.h @@ -1,9 +1,8 @@ //===-- BTF.h --------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -18,7 +17,7 @@ /// /// The binary layout for .BTF.ext section: /// struct ExtHeader -/// FuncInfo and LineInfo subsections +/// FuncInfo, LineInfo, OffsetReloc and ExternReloc subsections /// The FuncInfo subsection is defined as below: /// BTFFuncInfo Size /// struct SecFuncInfo for ELF section #1 @@ -33,6 +32,20 @@ /// struct SecLineInfo for ELF section #2 /// A number of struct BPFLineInfo for ELF section #2 /// ... +/// The OffsetReloc subsection is defined as below: +/// BPFOffsetReloc Size +/// struct SecOffsetReloc for ELF section #1 +/// A number of struct BPFOffsetReloc for ELF section #1 +/// struct SecOffsetReloc for ELF section #2 +/// A number of struct BPFOffsetReloc for ELF section #2 +/// ... +/// The ExternReloc subsection is defined as below: +/// BPFExternReloc Size +/// struct SecExternReloc for ELF section #1 +/// A number of struct BPFExternReloc for ELF section #1 +/// struct SecExternReloc for ELF section #2 +/// A number of struct BPFExternReloc for ELF section #2 +/// ... /// /// The section formats are also defined at /// https://github.com/torvalds/linux/blob/master/include/uapi/linux/btf.h @@ -50,16 +63,21 @@ enum : uint32_t { MAGIC = 0xeB9F, VERSION = 1 }; /// Sizes in bytes of various things in the BTF format. enum { HeaderSize = 24, - ExtHeaderSize = 24, + ExtHeaderSize = 40, CommonTypeSize = 12, BTFArraySize = 12, BTFEnumSize = 8, BTFMemberSize = 12, BTFParamSize = 8, + BTFDataSecVarSize = 12, SecFuncInfoSize = 8, SecLineInfoSize = 8, + SecOffsetRelocSize = 8, + SecExternRelocSize = 8, BPFFuncInfoSize = 8, - BPFLineInfoSize = 16 + BPFLineInfoSize = 16, + BPFOffsetRelocSize = 12, + BPFExternRelocSize = 8, }; /// The .BTF section header definition. @@ -77,7 +95,7 @@ struct Header { }; enum : uint32_t { - MAX_VLEN = 0xffff ///< Max # of struct/union/enum members or func args + MAX_VLEN = 0xffff ///< Max # of struct/union/enum members or func args }; enum TypeKinds : uint8_t { @@ -104,7 +122,7 @@ struct CommonType { /// "Size" tells the size of the type it is describing. /// /// "Type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - /// FUNC and FUNC_PROTO. + /// FUNC, FUNC_PROTO and VAR. /// "Type" is a type_id referring to another type. union { uint32_t Size; @@ -122,7 +140,11 @@ struct CommonType { // BTF_INT_BITS(VAL) : ((VAL) & 0x000000ff) /// Attributes stored in the INT_ENCODING. -enum : uint8_t { INT_SIGNED = (1 << 0), INT_CHAR = (1 << 1), INT_BOOL = (1 << 2) }; +enum : uint8_t { + INT_SIGNED = (1 << 0), + INT_CHAR = (1 << 1), + INT_BOOL = (1 << 2) +}; /// BTF_KIND_ENUM is followed by multiple "struct BTFEnum". /// The exact number of btf_enum is stored in the vlen (of the @@ -163,6 +185,23 @@ struct BTFParam { uint32_t Type; }; +/// Variable scoping information. +enum : uint8_t { + VAR_STATIC = 0, ///< Linkage: InternalLinkage + VAR_GLOBAL_ALLOCATED = 1, ///< Linkage: ExternalLinkage + VAR_GLOBAL_TENTATIVE = 2, ///< Linkage: CommonLinkage + VAR_GLOBAL_EXTERNAL = 3, ///< Linkage: ExternalLinkage +}; + +/// BTF_KIND_DATASEC are followed by multiple "struct BTFDataSecVar". +/// The exist number of BTFDataSec is stored in the vlen (of the info +/// in "struct CommonType"). +struct BTFDataSec { + uint32_t Type; ///< A BTF_KIND_VAR type + uint32_t Offset; ///< In-section offset + uint32_t Size; ///< Occupied memory size +}; + /// The .BTF.ext section header definition. struct ExtHeader { uint16_t Magic; @@ -170,10 +209,14 @@ struct ExtHeader { uint8_t Flags; uint32_t HdrLen; - uint32_t FuncInfoOff; ///< Offset of func info section - uint32_t FuncInfoLen; ///< Length of func info section - uint32_t LineInfoOff; ///< Offset of line info section - uint32_t LineInfoLen; ///< Length of line info section + uint32_t FuncInfoOff; ///< Offset of func info section + uint32_t FuncInfoLen; ///< Length of func info section + uint32_t LineInfoOff; ///< Offset of line info section + uint32_t LineInfoLen; ///< Length of line info section + uint32_t OffsetRelocOff; ///< Offset of offset reloc section + uint32_t OffsetRelocLen; ///< Length of offset reloc section + uint32_t ExternRelocOff; ///< Offset of extern reloc section + uint32_t ExternRelocLen; ///< Length of extern reloc section }; /// Specifying one function info. @@ -199,10 +242,35 @@ struct BPFLineInfo { /// Specifying line info's in one section. struct SecLineInfo { - uint32_t SecNameOff; ///< Section name index in the .BTF string tble + uint32_t SecNameOff; ///< Section name index in the .BTF string table uint32_t NumLineInfo; ///< Number of line info's in this section }; +/// Specifying one offset relocation. +struct BPFOffsetReloc { + uint32_t InsnOffset; ///< Byte offset in this section + uint32_t TypeID; ///< TypeID for the relocation + uint32_t OffsetNameOff; ///< The string to traverse types +}; + +/// Specifying offset relocation's in one section. +struct SecOffsetReloc { + uint32_t SecNameOff; ///< Section name index in the .BTF string table + uint32_t NumOffsetReloc; ///< Number of offset reloc's in this section +}; + +/// Specifying one offset relocation. +struct BPFExternReloc { + uint32_t InsnOffset; ///< Byte offset in this section + uint32_t ExternNameOff; ///< The string for external variable +}; + +/// Specifying extern relocation's in one section. +struct SecExternReloc { + uint32_t SecNameOff; ///< Section name index in the .BTF string table + uint32_t NumExternReloc; ///< Number of extern reloc's in this section +}; + } // End namespace BTF. } // End namespace llvm. diff --git a/contrib/llvm/lib/Target/BPF/BTFDebug.cpp b/contrib/llvm/lib/Target/BPF/BTFDebug.cpp index 96efea4ba8ee..fa35c6619e21 100644 --- a/contrib/llvm/lib/Target/BPF/BTFDebug.cpp +++ b/contrib/llvm/lib/Target/BPF/BTFDebug.cpp @@ -1,9 +1,8 @@ //===- BTFDebug.cpp - BTF Generator ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,6 +11,9 @@ //===----------------------------------------------------------------------===// #include "BTFDebug.h" +#include "BPF.h" +#include "BPFCORE.h" +#include "MCTargetDesc/BPFMCTargetDesc.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -19,8 +21,7 @@ #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCStreamer.h" -#include <fstream> -#include <sstream> +#include "llvm/Support/LineIterator.h" using namespace llvm; @@ -39,8 +40,9 @@ void BTFTypeBase::emitType(MCStreamer &OS) { OS.EmitIntValue(BTFType.Size, 4); } -BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag) - : DTy(DTy) { +BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag, + bool NeedsFixup) + : DTy(DTy), NeedsFixup(NeedsFixup) { switch (Tag) { case dwarf::DW_TAG_pointer_type: Kind = BTF::BTF_KIND_PTR; @@ -64,10 +66,17 @@ BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag) } void BTFTypeDerived::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + BTFType.NameOff = BDebug.addString(DTy->getName()); + if (NeedsFixup) + return; + // The base type for PTR/CONST/VOLATILE could be void. - const DIType *ResolvedType = DTy->getBaseType().resolve(); + const DIType *ResolvedType = DTy->getBaseType(); if (!ResolvedType) { assert((Kind == BTF::BTF_KIND_PTR || Kind == BTF::BTF_KIND_CONST || Kind == BTF::BTF_KIND_VOLATILE) && @@ -80,6 +89,10 @@ void BTFTypeDerived::completeType(BTFDebug &BDebug) { void BTFTypeDerived::emitType(MCStreamer &OS) { BTFTypeBase::emitType(OS); } +void BTFTypeDerived::setPointeeType(uint32_t PointeeType) { + BTFType.Type = PointeeType; +} + /// Represent a struct/union forward declaration. BTFTypeFwd::BTFTypeFwd(StringRef Name, bool IsUnion) : Name(Name) { Kind = BTF::BTF_KIND_FWD; @@ -88,6 +101,10 @@ BTFTypeFwd::BTFTypeFwd(StringRef Name, bool IsUnion) : Name(Name) { } void BTFTypeFwd::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + BTFType.NameOff = BDebug.addString(Name); } @@ -121,6 +138,10 @@ BTFTypeInt::BTFTypeInt(uint32_t Encoding, uint32_t SizeInBits, } void BTFTypeInt::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + BTFType.NameOff = BDebug.addString(Name); } @@ -137,6 +158,10 @@ BTFTypeEnum::BTFTypeEnum(const DICompositeType *ETy, uint32_t VLen) : ETy(ETy) { } void BTFTypeEnum::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + BTFType.NameOff = BDebug.addString(ETy->getName()); DINodeArray Elements = ETy->getElements(); @@ -159,45 +184,29 @@ void BTFTypeEnum::emitType(MCStreamer &OS) { } } -BTFTypeArray::BTFTypeArray(const DICompositeType *ATy) : ATy(ATy) { +BTFTypeArray::BTFTypeArray(uint32_t ElemTypeId, uint32_t ElemSize, + uint32_t NumElems) + : ElemSize(ElemSize) { Kind = BTF::BTF_KIND_ARRAY; + BTFType.NameOff = 0; BTFType.Info = Kind << 24; + BTFType.Size = 0; + + ArrayInfo.ElemType = ElemTypeId; + ArrayInfo.Nelems = NumElems; } -/// Represent a BTF array. BTF does not record array dimensions, -/// so conceptually a BTF array is a one-dimensional array. +/// Represent a BTF array. void BTFTypeArray::completeType(BTFDebug &BDebug) { - BTFType.NameOff = BDebug.addString(ATy->getName()); - BTFType.Size = 0; - - auto *BaseType = ATy->getBaseType().resolve(); - ArrayInfo.ElemType = BDebug.getTypeId(BaseType); + if (IsCompleted) + return; + IsCompleted = true; // The IR does not really have a type for the index. // A special type for array index should have been // created during initial type traversal. Just // retrieve that type id. ArrayInfo.IndexType = BDebug.getArrayIndexTypeId(); - - // Get the number of array elements. - // If the array size is 0, set the number of elements as 0. - // Otherwise, recursively traverse the base types to - // find the element size. The number of elements is - // the totoal array size in bits divided by - // element size in bits. - uint64_t ArraySizeInBits = ATy->getSizeInBits(); - if (!ArraySizeInBits) { - ArrayInfo.Nelems = 0; - } else { - uint32_t BaseTypeSize = BaseType->getSizeInBits(); - while (!BaseTypeSize) { - const auto *DDTy = cast<DIDerivedType>(BaseType); - BaseType = DDTy->getBaseType().resolve(); - assert(BaseType); - BaseTypeSize = BaseType->getSizeInBits(); - } - ArrayInfo.Nelems = ATy->getSizeInBits() / BaseTypeSize; - } } void BTFTypeArray::emitType(MCStreamer &OS) { @@ -207,6 +216,12 @@ void BTFTypeArray::emitType(MCStreamer &OS) { OS.EmitIntValue(ArrayInfo.Nelems, 4); } +void BTFTypeArray::getLocInfo(uint32_t Loc, uint32_t &LocOffset, + uint32_t &ElementTypeId) { + ElementTypeId = ArrayInfo.ElemType; + LocOffset = Loc * ElemSize; +} + /// Represent either a struct or a union. BTFTypeStruct::BTFTypeStruct(const DICompositeType *STy, bool IsStruct, bool HasBitField, uint32_t Vlen) @@ -217,6 +232,10 @@ BTFTypeStruct::BTFTypeStruct(const DICompositeType *STy, bool IsStruct, } void BTFTypeStruct::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + BTFType.NameOff = BDebug.addString(STy->getName()); // Add struct/union members. @@ -232,7 +251,7 @@ void BTFTypeStruct::completeType(BTFDebug &BDebug) { } else { BTFMember.Offset = DDTy->getOffsetInBits(); } - BTFMember.Type = BDebug.getTypeId(DDTy->getBaseType().resolve()); + BTFMember.Type = BDebug.getTypeId(DDTy->getBaseType()); Members.push_back(BTFMember); } } @@ -247,6 +266,17 @@ void BTFTypeStruct::emitType(MCStreamer &OS) { } } +std::string BTFTypeStruct::getName() { return STy->getName(); } + +void BTFTypeStruct::getMemberInfo(uint32_t Loc, uint32_t &MemberOffset, + uint32_t &MemberType) { + MemberType = Members[Loc].Type; + MemberOffset = + HasBitField ? Members[Loc].Offset & 0xffffff : Members[Loc].Offset; +} + +uint32_t BTFTypeStruct::getStructSize() { return STy->getSizeInBits() >> 3; } + /// The Func kind represents both subprogram and pointee of function /// pointers. If the FuncName is empty, it represents a pointee of function /// pointer. Otherwise, it represents a subprogram. The func arg names @@ -261,8 +291,12 @@ BTFTypeFuncProto::BTFTypeFuncProto( } void BTFTypeFuncProto::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + DITypeRefArray Elements = STy->getTypeArray(); - auto RetType = Elements[0].resolve(); + auto RetType = Elements[0]; BTFType.Type = RetType ? BDebug.getTypeId(RetType) : 0; BTFType.NameOff = 0; @@ -270,7 +304,7 @@ void BTFTypeFuncProto::completeType(BTFDebug &BDebug) { // to represent the vararg, encode the NameOff/Type to be 0. for (unsigned I = 1, N = Elements.size(); I < N; ++I) { struct BTF::BTFParam Param; - auto Element = Elements[I].resolve(); + auto Element = Elements[I]; if (Element) { Param.NameOff = BDebug.addString(FuncArgNames[I]); Param.Type = BDebug.getTypeId(Element); @@ -298,11 +332,54 @@ BTFTypeFunc::BTFTypeFunc(StringRef FuncName, uint32_t ProtoTypeId) } void BTFTypeFunc::completeType(BTFDebug &BDebug) { + if (IsCompleted) + return; + IsCompleted = true; + BTFType.NameOff = BDebug.addString(Name); } void BTFTypeFunc::emitType(MCStreamer &OS) { BTFTypeBase::emitType(OS); } +BTFKindVar::BTFKindVar(StringRef VarName, uint32_t TypeId, uint32_t VarInfo) + : Name(VarName) { + Kind = BTF::BTF_KIND_VAR; + BTFType.Info = Kind << 24; + BTFType.Type = TypeId; + Info = VarInfo; +} + +void BTFKindVar::completeType(BTFDebug &BDebug) { + BTFType.NameOff = BDebug.addString(Name); +} + +void BTFKindVar::emitType(MCStreamer &OS) { + BTFTypeBase::emitType(OS); + OS.EmitIntValue(Info, 4); +} + +BTFKindDataSec::BTFKindDataSec(AsmPrinter *AsmPrt, std::string SecName) + : Asm(AsmPrt), Name(SecName) { + Kind = BTF::BTF_KIND_DATASEC; + BTFType.Info = Kind << 24; + BTFType.Size = 0; +} + +void BTFKindDataSec::completeType(BTFDebug &BDebug) { + BTFType.NameOff = BDebug.addString(Name); + BTFType.Info |= Vars.size(); +} + +void BTFKindDataSec::emitType(MCStreamer &OS) { + BTFTypeBase::emitType(OS); + + for (const auto &V : Vars) { + OS.EmitIntValue(std::get<0>(V), 4); + Asm->EmitLabelReference(std::get<1>(V), 4); + OS.EmitIntValue(std::get<2>(V), 4); + } +} + uint32_t BTFStringTable::addString(StringRef S) { // Check whether the string already exists. for (auto &OffsetM : OffsetToIdMap) { @@ -319,15 +396,18 @@ uint32_t BTFStringTable::addString(StringRef S) { BTFDebug::BTFDebug(AsmPrinter *AP) : DebugHandlerBase(AP), OS(*Asm->OutStreamer), SkipInstruction(false), - LineInfoGenerated(false), SecNameOff(0), ArrayIndexTypeId(0) { + LineInfoGenerated(false), SecNameOff(0), ArrayIndexTypeId(0), + MapDefNotCollected(true) { addString("\0"); } -void BTFDebug::addType(std::unique_ptr<BTFTypeBase> TypeEntry, - const DIType *Ty) { +uint32_t BTFDebug::addType(std::unique_ptr<BTFTypeBase> TypeEntry, + const DIType *Ty) { TypeEntry->setId(TypeEntries.size() + 1); - DIToIdMap[Ty] = TypeEntry->getId(); + uint32_t Id = TypeEntry->getId(); + DIToIdMap[Ty] = Id; TypeEntries.push_back(std::move(TypeEntry)); + return Id; } uint32_t BTFDebug::addType(std::unique_ptr<BTFTypeBase> TypeEntry) { @@ -337,7 +417,7 @@ uint32_t BTFDebug::addType(std::unique_ptr<BTFTypeBase> TypeEntry) { return Id; } -void BTFDebug::visitBasicType(const DIBasicType *BTy) { +void BTFDebug::visitBasicType(const DIBasicType *BTy, uint32_t &TypeId) { // Only int types are supported in BTF. uint32_t Encoding = BTy->getEncoding(); if (Encoding != dwarf::DW_ATE_boolean && Encoding != dwarf::DW_ATE_signed && @@ -350,7 +430,7 @@ void BTFDebug::visitBasicType(const DIBasicType *BTy) { // DIToIdMap for cross-type reference check. auto TypeEntry = llvm::make_unique<BTFTypeInt>( Encoding, BTy->getSizeInBits(), BTy->getOffsetInBits(), BTy->getName()); - addType(std::move(TypeEntry), BTy); + TypeId = addType(std::move(TypeEntry), BTy); } /// Handle subprogram or subroutine types. @@ -371,16 +451,17 @@ void BTFDebug::visitSubroutineType( if (ForSubprog) TypeId = addType(std::move(TypeEntry)); // For subprogram else - addType(std::move(TypeEntry), STy); // For func ptr + TypeId = addType(std::move(TypeEntry), STy); // For func ptr // Visit return type and func arg types. for (const auto Element : Elements) { - visitTypeEntry(Element.resolve()); + visitTypeEntry(Element); } } /// Handle structure/union types. -void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct) { +void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct, + uint32_t &TypeId) { const DINodeArray Elements = CTy->getElements(); uint32_t VLen = Elements.size(); if (VLen > BTF::MAX_VLEN) @@ -398,16 +479,49 @@ void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct) { auto TypeEntry = llvm::make_unique<BTFTypeStruct>(CTy, IsStruct, HasBitField, VLen); - addType(std::move(TypeEntry), CTy); + StructTypes.push_back(TypeEntry.get()); + TypeId = addType(std::move(TypeEntry), CTy); // Visit all struct members. for (const auto *Element : Elements) visitTypeEntry(cast<DIDerivedType>(Element)); } -void BTFDebug::visitArrayType(const DICompositeType *CTy) { - auto TypeEntry = llvm::make_unique<BTFTypeArray>(CTy); - addType(std::move(TypeEntry), CTy); +void BTFDebug::visitArrayType(const DICompositeType *CTy, uint32_t &TypeId) { + // Visit array element type. + uint32_t ElemTypeId, ElemSize; + const DIType *ElemType = CTy->getBaseType(); + visitTypeEntry(ElemType, ElemTypeId, false, false); + ElemSize = ElemType->getSizeInBits() >> 3; + + if (!CTy->getSizeInBits()) { + auto TypeEntry = llvm::make_unique<BTFTypeArray>(ElemTypeId, 0, 0); + ArrayTypes.push_back(TypeEntry.get()); + ElemTypeId = addType(std::move(TypeEntry), CTy); + } else { + // Visit array dimensions. + DINodeArray Elements = CTy->getElements(); + for (int I = Elements.size() - 1; I >= 0; --I) { + if (auto *Element = dyn_cast_or_null<DINode>(Elements[I])) + if (Element->getTag() == dwarf::DW_TAG_subrange_type) { + const DISubrange *SR = cast<DISubrange>(Element); + auto *CI = SR->getCount().dyn_cast<ConstantInt *>(); + int64_t Count = CI->getSExtValue(); + + auto TypeEntry = + llvm::make_unique<BTFTypeArray>(ElemTypeId, ElemSize, Count); + ArrayTypes.push_back(TypeEntry.get()); + if (I == 0) + ElemTypeId = addType(std::move(TypeEntry), CTy); + else + ElemTypeId = addType(std::move(TypeEntry)); + ElemSize = ElemSize * Count; + } + } + } + + // The array TypeId is the type id of the outermost dimension. + TypeId = ElemTypeId; // The IR does not have a type for array index while BTF wants one. // So create an array index type if there is none. @@ -416,85 +530,162 @@ void BTFDebug::visitArrayType(const DICompositeType *CTy) { 0, "__ARRAY_SIZE_TYPE__"); ArrayIndexTypeId = addType(std::move(TypeEntry)); } - - // Visit array element type. - visitTypeEntry(CTy->getBaseType().resolve()); } -void BTFDebug::visitEnumType(const DICompositeType *CTy) { +void BTFDebug::visitEnumType(const DICompositeType *CTy, uint32_t &TypeId) { DINodeArray Elements = CTy->getElements(); uint32_t VLen = Elements.size(); if (VLen > BTF::MAX_VLEN) return; auto TypeEntry = llvm::make_unique<BTFTypeEnum>(CTy, VLen); - addType(std::move(TypeEntry), CTy); + TypeId = addType(std::move(TypeEntry), CTy); // No need to visit base type as BTF does not encode it. } /// Handle structure/union forward declarations. -void BTFDebug::visitFwdDeclType(const DICompositeType *CTy, bool IsUnion) { +void BTFDebug::visitFwdDeclType(const DICompositeType *CTy, bool IsUnion, + uint32_t &TypeId) { auto TypeEntry = llvm::make_unique<BTFTypeFwd>(CTy->getName(), IsUnion); - addType(std::move(TypeEntry), CTy); + TypeId = addType(std::move(TypeEntry), CTy); } /// Handle structure, union, array and enumeration types. -void BTFDebug::visitCompositeType(const DICompositeType *CTy) { +void BTFDebug::visitCompositeType(const DICompositeType *CTy, + uint32_t &TypeId) { auto Tag = CTy->getTag(); if (Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type) { // Handle forward declaration differently as it does not have members. if (CTy->isForwardDecl()) - visitFwdDeclType(CTy, Tag == dwarf::DW_TAG_union_type); + visitFwdDeclType(CTy, Tag == dwarf::DW_TAG_union_type, TypeId); else - visitStructType(CTy, Tag == dwarf::DW_TAG_structure_type); + visitStructType(CTy, Tag == dwarf::DW_TAG_structure_type, TypeId); } else if (Tag == dwarf::DW_TAG_array_type) - visitArrayType(CTy); + visitArrayType(CTy, TypeId); else if (Tag == dwarf::DW_TAG_enumeration_type) - visitEnumType(CTy); + visitEnumType(CTy, TypeId); } /// Handle pointer, typedef, const, volatile, restrict and member types. -void BTFDebug::visitDerivedType(const DIDerivedType *DTy) { +void BTFDebug::visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId, + bool CheckPointer, bool SeenPointer) { unsigned Tag = DTy->getTag(); + /// Try to avoid chasing pointees, esp. structure pointees which may + /// unnecessary bring in a lot of types. + if (CheckPointer && !SeenPointer) { + SeenPointer = Tag == dwarf::DW_TAG_pointer_type; + } + + if (CheckPointer && SeenPointer) { + const DIType *Base = DTy->getBaseType(); + if (Base) { + if (const auto *CTy = dyn_cast<DICompositeType>(Base)) { + auto CTag = CTy->getTag(); + if ((CTag == dwarf::DW_TAG_structure_type || + CTag == dwarf::DW_TAG_union_type) && + !CTy->isForwardDecl()) { + /// Find a candidate, generate a fixup. Later on the struct/union + /// pointee type will be replaced with either a real type or + /// a forward declaration. + auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag, true); + auto &Fixup = FixupDerivedTypes[CTy->getName()]; + Fixup.first = CTag == dwarf::DW_TAG_union_type; + Fixup.second.push_back(TypeEntry.get()); + TypeId = addType(std::move(TypeEntry), DTy); + return; + } + } + } + } + if (Tag == dwarf::DW_TAG_pointer_type || Tag == dwarf::DW_TAG_typedef || Tag == dwarf::DW_TAG_const_type || Tag == dwarf::DW_TAG_volatile_type || Tag == dwarf::DW_TAG_restrict_type) { - auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag); - addType(std::move(TypeEntry), DTy); + auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag, false); + TypeId = addType(std::move(TypeEntry), DTy); } else if (Tag != dwarf::DW_TAG_member) { return; } // Visit base type of pointer, typedef, const, volatile, restrict or // struct/union member. - visitTypeEntry(DTy->getBaseType().resolve()); + uint32_t TempTypeId = 0; + if (Tag == dwarf::DW_TAG_member) + visitTypeEntry(DTy->getBaseType(), TempTypeId, true, false); + else + visitTypeEntry(DTy->getBaseType(), TempTypeId, CheckPointer, SeenPointer); } -void BTFDebug::visitTypeEntry(const DIType *Ty) { - if (!Ty || DIToIdMap.find(Ty) != DIToIdMap.end()) +void BTFDebug::visitTypeEntry(const DIType *Ty, uint32_t &TypeId, + bool CheckPointer, bool SeenPointer) { + if (!Ty || DIToIdMap.find(Ty) != DIToIdMap.end()) { + TypeId = DIToIdMap[Ty]; return; + } - uint32_t TypeId; if (const auto *BTy = dyn_cast<DIBasicType>(Ty)) - visitBasicType(BTy); + visitBasicType(BTy, TypeId); else if (const auto *STy = dyn_cast<DISubroutineType>(Ty)) visitSubroutineType(STy, false, std::unordered_map<uint32_t, StringRef>(), TypeId); else if (const auto *CTy = dyn_cast<DICompositeType>(Ty)) - visitCompositeType(CTy); + visitCompositeType(CTy, TypeId); else if (const auto *DTy = dyn_cast<DIDerivedType>(Ty)) - visitDerivedType(DTy); + visitDerivedType(DTy, TypeId, CheckPointer, SeenPointer); else llvm_unreachable("Unknown DIType"); } +void BTFDebug::visitTypeEntry(const DIType *Ty) { + uint32_t TypeId; + visitTypeEntry(Ty, TypeId, false, false); +} + +void BTFDebug::visitMapDefType(const DIType *Ty, uint32_t &TypeId) { + if (!Ty || DIToIdMap.find(Ty) != DIToIdMap.end()) { + TypeId = DIToIdMap[Ty]; + return; + } + + // MapDef type is a struct type + const auto *CTy = dyn_cast<DICompositeType>(Ty); + if (!CTy) + return; + + auto Tag = CTy->getTag(); + if (Tag != dwarf::DW_TAG_structure_type || CTy->isForwardDecl()) + return; + + // Record this type + const DINodeArray Elements = CTy->getElements(); + bool HasBitField = false; + for (const auto *Element : Elements) { + auto E = cast<DIDerivedType>(Element); + if (E->isBitField()) { + HasBitField = true; + break; + } + } + + auto TypeEntry = + llvm::make_unique<BTFTypeStruct>(CTy, true, HasBitField, Elements.size()); + StructTypes.push_back(TypeEntry.get()); + TypeId = addType(std::move(TypeEntry), CTy); + + // Visit all struct members + for (const auto *Element : Elements) { + const auto *MemberType = cast<DIDerivedType>(Element); + visitTypeEntry(MemberType->getBaseType()); + } +} + /// Read file contents from the actual file or from the source std::string BTFDebug::populateFileContent(const DISubprogram *SP) { auto File = SP->getFile(); std::string FileName; - if (File->getDirectory().size()) + if (!File->getFilename().startswith("/") && File->getDirectory().size()) FileName = File->getDirectory().str() + "/" + File->getFilename().str(); else FileName = File->getFilename(); @@ -507,16 +698,16 @@ std::string BTFDebug::populateFileContent(const DISubprogram *SP) { std::string Line; Content.push_back(Line); // Line 0 for empty string + std::unique_ptr<MemoryBuffer> Buf; auto Source = File->getSource(); - if (Source) { - std::istringstream InputString(Source.getValue()); - while (std::getline(InputString, Line)) - Content.push_back(Line); - } else { - std::ifstream InputFile(FileName); - while (std::getline(InputFile, Line)) - Content.push_back(Line); - } + if (Source) + Buf = MemoryBuffer::getMemBufferCopy(*Source); + else if (ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr = + MemoryBuffer::getFile(FileName)) + Buf = std::move(*BufOrErr); + if (Buf) + for (line_iterator I(*Buf, false), E; I != E; ++I) + Content.push_back(*I); FileContent[FileName] = Content; return FileName; @@ -547,6 +738,10 @@ void BTFDebug::emitCommonHeader() { } void BTFDebug::emitBTFSection() { + // Do not emit section if no types and only "" string. + if (!TypeEntries.size() && StringTable.getSize() == 1) + return; + MCContext &Ctx = OS.getContext(); OS.SwitchSection(Ctx.getELFSection(".BTF", ELF::SHT_PROGBITS, 0)); @@ -579,6 +774,11 @@ void BTFDebug::emitBTFSection() { } void BTFDebug::emitBTFExtSection() { + // Do not emit section if empty FuncInfoTable and LineInfoTable. + if (!FuncInfoTable.size() && !LineInfoTable.size() && + !OffsetRelocTable.size() && !ExternRelocTable.size()) + return; + MCContext &Ctx = OS.getContext(); OS.SwitchSection(Ctx.getELFSection(".BTF.ext", ELF::SHT_PROGBITS, 0)); @@ -588,6 +788,8 @@ void BTFDebug::emitBTFExtSection() { // Account for FuncInfo/LineInfo record size as well. uint32_t FuncLen = 4, LineLen = 4; + // Do not account for optional OffsetReloc/ExternReloc. + uint32_t OffsetRelocLen = 0, ExternRelocLen = 0; for (const auto &FuncSec : FuncInfoTable) { FuncLen += BTF::SecFuncInfoSize; FuncLen += FuncSec.second.size() * BTF::BPFFuncInfoSize; @@ -596,11 +798,28 @@ void BTFDebug::emitBTFExtSection() { LineLen += BTF::SecLineInfoSize; LineLen += LineSec.second.size() * BTF::BPFLineInfoSize; } + for (const auto &OffsetRelocSec : OffsetRelocTable) { + OffsetRelocLen += BTF::SecOffsetRelocSize; + OffsetRelocLen += OffsetRelocSec.second.size() * BTF::BPFOffsetRelocSize; + } + for (const auto &ExternRelocSec : ExternRelocTable) { + ExternRelocLen += BTF::SecExternRelocSize; + ExternRelocLen += ExternRelocSec.second.size() * BTF::BPFExternRelocSize; + } + + if (OffsetRelocLen) + OffsetRelocLen += 4; + if (ExternRelocLen) + ExternRelocLen += 4; OS.EmitIntValue(0, 4); OS.EmitIntValue(FuncLen, 4); OS.EmitIntValue(FuncLen, 4); OS.EmitIntValue(LineLen, 4); + OS.EmitIntValue(FuncLen + LineLen, 4); + OS.EmitIntValue(OffsetRelocLen, 4); + OS.EmitIntValue(FuncLen + LineLen + OffsetRelocLen, 4); + OS.EmitIntValue(ExternRelocLen, 4); // Emit func_info table. OS.AddComment("FuncInfo"); @@ -633,6 +852,39 @@ void BTFDebug::emitBTFExtSection() { OS.EmitIntValue(LineInfo.LineNum << 10 | LineInfo.ColumnNum, 4); } } + + // Emit offset reloc table. + if (OffsetRelocLen) { + OS.AddComment("OffsetReloc"); + OS.EmitIntValue(BTF::BPFOffsetRelocSize, 4); + for (const auto &OffsetRelocSec : OffsetRelocTable) { + OS.AddComment("Offset reloc section string offset=" + + std::to_string(OffsetRelocSec.first)); + OS.EmitIntValue(OffsetRelocSec.first, 4); + OS.EmitIntValue(OffsetRelocSec.second.size(), 4); + for (const auto &OffsetRelocInfo : OffsetRelocSec.second) { + Asm->EmitLabelReference(OffsetRelocInfo.Label, 4); + OS.EmitIntValue(OffsetRelocInfo.TypeID, 4); + OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4); + } + } + } + + // Emit extern reloc table. + if (ExternRelocLen) { + OS.AddComment("ExternReloc"); + OS.EmitIntValue(BTF::BPFExternRelocSize, 4); + for (const auto &ExternRelocSec : ExternRelocTable) { + OS.AddComment("Extern reloc section string offset=" + + std::to_string(ExternRelocSec.first)); + OS.EmitIntValue(ExternRelocSec.first, 4); + OS.EmitIntValue(ExternRelocSec.second.size(), 4); + for (const auto &ExternRelocInfo : ExternRelocSec.second) { + Asm->EmitLabelReference(ExternRelocInfo.Label, 4); + OS.EmitIntValue(ExternRelocInfo.ExternNameOff, 4); + } + } + } } void BTFDebug::beginFunctionImpl(const MachineFunction *MF) { @@ -645,18 +897,42 @@ void BTFDebug::beginFunctionImpl(const MachineFunction *MF) { } SkipInstruction = false; + // Collect MapDef types. Map definition needs to collect + // pointee types. Do it first. Otherwise, for the following + // case: + // struct m { ...}; + // struct t { + // struct m *key; + // }; + // foo(struct t *arg); + // + // struct mapdef { + // ... + // struct m *key; + // ... + // } __attribute__((section(".maps"))) hash_map; + // + // If subroutine foo is traversed first, a type chain + // "ptr->struct m(fwd)" will be created and later on + // when traversing mapdef, since "ptr->struct m" exists, + // the traversal of "struct m" will be omitted. + if (MapDefNotCollected) { + processGlobals(true); + MapDefNotCollected = false; + } + // Collect all types locally referenced in this function. // Use RetainedNodes so we can collect all argument names // even if the argument is not used. std::unordered_map<uint32_t, StringRef> FuncArgNames; for (const DINode *DN : SP->getRetainedNodes()) { if (const auto *DV = dyn_cast<DILocalVariable>(DN)) { - visitTypeEntry(DV->getType().resolve()); - // Collect function arguments for subprogram func type. uint32_t Arg = DV->getArg(); - if (Arg) + if (Arg) { + visitTypeEntry(DV->getType()); FuncArgNames[Arg] = DV->getName(); + } } } @@ -669,6 +945,9 @@ void BTFDebug::beginFunctionImpl(const MachineFunction *MF) { llvm::make_unique<BTFTypeFunc>(SP->getName(), ProtoTypeId); uint32_t FuncTypeId = addType(std::move(FuncTypeEntry)); + for (const auto &TypeEntry : TypeEntries) + TypeEntry->completeType(*this); + // Construct funcinfo and the first lineinfo for the function. MCSymbol *FuncLabel = Asm->getFunctionBegin(); BTFFuncInfo FuncInfo; @@ -691,6 +970,133 @@ void BTFDebug::endFunctionImpl(const MachineFunction *MF) { SecNameOff = 0; } +/// On-demand populate struct types as requested from abstract member +/// accessing. +unsigned BTFDebug::populateStructType(const DIType *Ty) { + unsigned Id; + visitTypeEntry(Ty, Id, false, false); + for (const auto &TypeEntry : TypeEntries) + TypeEntry->completeType(*this); + return Id; +} + +// Find struct/array debuginfo types given a type id. +void BTFDebug::setTypeFromId(uint32_t TypeId, BTFTypeStruct **PrevStructType, + BTFTypeArray **PrevArrayType) { + for (const auto &StructType : StructTypes) { + if (StructType->getId() == TypeId) { + *PrevStructType = StructType; + return; + } + } + for (const auto &ArrayType : ArrayTypes) { + if (ArrayType->getId() == TypeId) { + *PrevArrayType = ArrayType; + return; + } + } +} + +/// Generate a struct member offset relocation. +void BTFDebug::generateOffsetReloc(const MachineInstr *MI, + const MCSymbol *ORSym, DIType *RootTy, + StringRef AccessPattern) { + BTFTypeStruct *PrevStructType = nullptr; + BTFTypeArray *PrevArrayType = nullptr; + unsigned RootId = populateStructType(RootTy); + setTypeFromId(RootId, &PrevStructType, &PrevArrayType); + unsigned RootTySize = PrevStructType->getStructSize(); + + BTFOffsetReloc OffsetReloc; + OffsetReloc.Label = ORSym; + OffsetReloc.OffsetNameOff = addString(AccessPattern.drop_back()); + OffsetReloc.TypeID = RootId; + + uint32_t Start = 0, End = 0, Offset = 0; + bool FirstAccess = true; + for (auto C : AccessPattern) { + if (C != ':') { + End++; + } else { + std::string SubStr = AccessPattern.substr(Start, End - Start); + int Loc = std::stoi(SubStr); + + if (FirstAccess) { + Offset = Loc * RootTySize; + FirstAccess = false; + } else if (PrevStructType) { + uint32_t MemberOffset, MemberTypeId; + PrevStructType->getMemberInfo(Loc, MemberOffset, MemberTypeId); + + Offset += MemberOffset >> 3; + PrevStructType = nullptr; + setTypeFromId(MemberTypeId, &PrevStructType, &PrevArrayType); + } else if (PrevArrayType) { + uint32_t LocOffset, ElementTypeId; + PrevArrayType->getLocInfo(Loc, LocOffset, ElementTypeId); + + Offset += LocOffset; + PrevArrayType = nullptr; + setTypeFromId(ElementTypeId, &PrevStructType, &PrevArrayType); + } + Start = End + 1; + End = Start; + } + } + AccessOffsets[RootTy->getName().str() + ":" + AccessPattern.str()] = Offset; + OffsetRelocTable[SecNameOff].push_back(OffsetReloc); +} + +void BTFDebug::processLDimm64(const MachineInstr *MI) { + // If the insn is an LD_imm64, the following two cases + // will generate an .BTF.ext record. + // + // If the insn is "r2 = LD_imm64 @__BTF_...", + // add this insn into the .BTF.ext OffsetReloc subsection. + // Relocation looks like: + // . SecName: + // . InstOffset + // . TypeID + // . OffSetNameOff + // Later, the insn is replaced with "r2 = <offset>" + // where "<offset>" equals to the offset based on current + // type definitions. + // + // If the insn is "r2 = LD_imm64 @VAR" and VAR is + // a patchable external global, add this insn into the .BTF.ext + // ExternReloc subsection. + // Relocation looks like: + // . SecName: + // . InstOffset + // . ExternNameOff + // Later, the insn is replaced with "r2 = <value>" or + // "LD_imm64 r2, <value>" where "<value>" = 0. + + // check whether this is a candidate or not + const MachineOperand &MO = MI->getOperand(1); + if (MO.isGlobal()) { + const GlobalValue *GVal = MO.getGlobal(); + auto *GVar = dyn_cast<GlobalVariable>(GVal); + if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) { + MCSymbol *ORSym = OS.getContext().createTempSymbol(); + OS.EmitLabel(ORSym); + + MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index); + DIType *Ty = dyn_cast<DIType>(MDN); + generateOffsetReloc(MI, ORSym, Ty, GVar->getName()); + } else if (GVar && !GVar->hasInitializer() && GVar->hasExternalLinkage() && + GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) { + MCSymbol *ORSym = OS.getContext().createTempSymbol(); + OS.EmitLabel(ORSym); + + BTFExternReloc ExternReloc; + ExternReloc.Label = ORSym; + ExternReloc.ExternNameOff = addString(GVar->getName()); + ExternRelocTable[SecNameOff].push_back(ExternReloc); + } + } +} + void BTFDebug::beginInstruction(const MachineInstr *MI) { DebugHandlerBase::beginInstruction(MI); @@ -711,6 +1117,9 @@ void BTFDebug::beginInstruction(const MachineInstr *MI) { return; } + if (MI->getOpcode() == BPF::LD_imm64) + processLDimm64(MI); + // Skip this instruction if no DebugLoc or the DebugLoc // is the same as the previous instruction. const DebugLoc &DL = MI->getDebugLoc(); @@ -739,13 +1148,145 @@ void BTFDebug::beginInstruction(const MachineInstr *MI) { PrevInstLoc = DL; } -void BTFDebug::endModule() { +void BTFDebug::processGlobals(bool ProcessingMapDef) { // Collect all types referenced by globals. const Module *M = MMI->getModule(); - for (const DICompileUnit *CUNode : M->debug_compile_units()) { - for (const auto *GVE : CUNode->getGlobalVariables()) { - DIGlobalVariable *GV = GVE->getVariable(); - visitTypeEntry(GV->getType().resolve()); + for (const GlobalVariable &Global : M->globals()) { + // Ignore external globals for now. + if (!Global.hasInitializer() && Global.hasExternalLinkage()) + continue; + + // Decide the section name. + StringRef SecName; + if (Global.hasSection()) { + SecName = Global.getSection(); + } else { + // data, bss, or readonly sections + if (Global.isConstant()) + SecName = ".rodata"; + else + SecName = Global.getInitializer()->isZeroValue() ? ".bss" : ".data"; + } + + if (ProcessingMapDef != SecName.startswith(".maps")) + continue; + + SmallVector<DIGlobalVariableExpression *, 1> GVs; + Global.getDebugInfo(GVs); + uint32_t GVTypeId = 0; + for (auto *GVE : GVs) { + if (SecName.startswith(".maps")) + visitMapDefType(GVE->getVariable()->getType(), GVTypeId); + else + visitTypeEntry(GVE->getVariable()->getType(), GVTypeId, false, false); + break; + } + + // Only support the following globals: + // . static variables + // . non-static global variables with section attributes + // Essentially means: + // . .bcc/.data/.rodata DataSec entities only contain static data + // . Other DataSec entities contain static or initialized global data. + // Initialized global data are mostly used for finding map key/value type + // id's. Whether DataSec is readonly or not can be found from + // corresponding ELF section flags. + auto Linkage = Global.getLinkage(); + if (Linkage != GlobalValue::InternalLinkage && + (Linkage != GlobalValue::ExternalLinkage || !Global.hasSection())) + continue; + + uint32_t GVarInfo = Linkage == GlobalValue::ExternalLinkage + ? BTF::VAR_GLOBAL_ALLOCATED + : BTF::VAR_STATIC; + auto VarEntry = + llvm::make_unique<BTFKindVar>(Global.getName(), GVTypeId, GVarInfo); + uint32_t VarId = addType(std::move(VarEntry)); + + // Find or create a DataSec + if (DataSecEntries.find(SecName) == DataSecEntries.end()) { + DataSecEntries[SecName] = llvm::make_unique<BTFKindDataSec>(Asm, SecName); + } + + // Calculate symbol size + const DataLayout &DL = Global.getParent()->getDataLayout(); + uint32_t Size = DL.getTypeAllocSize(Global.getType()->getElementType()); + + DataSecEntries[SecName]->addVar(VarId, Asm->getSymbol(&Global), Size); + } +} + +/// Emit proper patchable instructions. +bool BTFDebug::InstLower(const MachineInstr *MI, MCInst &OutMI) { + if (MI->getOpcode() == BPF::LD_imm64) { + const MachineOperand &MO = MI->getOperand(1); + if (MO.isGlobal()) { + const GlobalValue *GVal = MO.getGlobal(); + auto *GVar = dyn_cast<GlobalVariable>(GVal); + if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) { + MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index); + DIType *Ty = dyn_cast<DIType>(MDN); + std::string TypeName = Ty->getName(); + int64_t Imm = AccessOffsets[TypeName + ":" + GVar->getName().str()]; + + // Emit "mov ri, <imm>" for abstract member accesses. + OutMI.setOpcode(BPF::MOV_ri); + OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); + OutMI.addOperand(MCOperand::createImm(Imm)); + return true; + } else if (GVar && !GVar->hasInitializer() && + GVar->hasExternalLinkage() && + GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) { + const IntegerType *IntTy = dyn_cast<IntegerType>(GVar->getValueType()); + assert(IntTy); + // For patchable externals, emit "LD_imm64, ri, 0" if the external + // variable is 64bit width, emit "mov ri, 0" otherwise. + if (IntTy->getBitWidth() == 64) + OutMI.setOpcode(BPF::LD_imm64); + else + OutMI.setOpcode(BPF::MOV_ri); + OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); + OutMI.addOperand(MCOperand::createImm(0)); + return true; + } + } + } + return false; +} + +void BTFDebug::endModule() { + // Collect MapDef globals if not collected yet. + if (MapDefNotCollected) { + processGlobals(true); + MapDefNotCollected = false; + } + + // Collect global types/variables except MapDef globals. + processGlobals(false); + for (auto &DataSec : DataSecEntries) + addType(std::move(DataSec.second)); + + // Fixups + for (auto &Fixup : FixupDerivedTypes) { + StringRef TypeName = Fixup.first; + bool IsUnion = Fixup.second.first; + + // Search through struct types + uint32_t StructTypeId = 0; + for (const auto &StructType : StructTypes) { + if (StructType->getName() == TypeName) { + StructTypeId = StructType->getId(); + break; + } + } + + if (StructTypeId == 0) { + auto FwdTypeEntry = llvm::make_unique<BTFTypeFwd>(TypeName, IsUnion); + StructTypeId = addType(std::move(FwdTypeEntry)); + } + + for (auto &DType : Fixup.second.second) { + DType->setPointeeType(StructTypeId); } } diff --git a/contrib/llvm/lib/Target/BPF/BTFDebug.h b/contrib/llvm/lib/Target/BPF/BTFDebug.h index afd4ed87f63d..6c0cdde17d9b 100644 --- a/contrib/llvm/lib/Target/BPF/BTFDebug.h +++ b/contrib/llvm/lib/Target/BPF/BTFDebug.h @@ -1,9 +1,8 @@ //===- BTFDebug.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -33,10 +32,12 @@ class MachineFunction; class BTFTypeBase { protected: uint8_t Kind; + bool IsCompleted; uint32_t Id; struct BTF::CommonType BTFType; public: + BTFTypeBase() : IsCompleted(false) {} virtual ~BTFTypeBase() = default; void setId(uint32_t Id) { this->Id = Id; } uint32_t getId() { return Id; } @@ -55,11 +56,13 @@ public: /// volatile, typedef and restrict. class BTFTypeDerived : public BTFTypeBase { const DIDerivedType *DTy; + bool NeedsFixup; public: - BTFTypeDerived(const DIDerivedType *Ty, unsigned Tag); + BTFTypeDerived(const DIDerivedType *Ty, unsigned Tag, bool NeedsFixup); void completeType(BTFDebug &BDebug); void emitType(MCStreamer &OS); + void setPointeeType(uint32_t PointeeType); }; /// Handle struct or union forward declaration. @@ -101,14 +104,15 @@ public: /// Handle array type. class BTFTypeArray : public BTFTypeBase { - const DICompositeType *ATy; + uint32_t ElemSize; struct BTF::BTFArray ArrayInfo; public: - BTFTypeArray(const DICompositeType *ATy); + BTFTypeArray(uint32_t ElemTypeId, uint32_t ElemSize, uint32_t NumElems); uint32_t getSize() { return BTFTypeBase::getSize() + BTF::BTFArraySize; } void completeType(BTFDebug &BDebug); void emitType(MCStreamer &OS); + void getLocInfo(uint32_t Loc, uint32_t &LocOffset, uint32_t &ElementTypeId); }; /// Handle struct/union type. @@ -125,6 +129,9 @@ public: } void completeType(BTFDebug &BDebug); void emitType(MCStreamer &OS); + std::string getName(); + void getMemberInfo(uint32_t Loc, uint32_t &Offset, uint32_t &MemberType); + uint32_t getStructSize(); }; /// Handle function pointer. @@ -154,6 +161,37 @@ public: void emitType(MCStreamer &OS); }; +/// Handle variable instances +class BTFKindVar : public BTFTypeBase { + StringRef Name; + uint32_t Info; + +public: + BTFKindVar(StringRef VarName, uint32_t TypeId, uint32_t VarInfo); + uint32_t getSize() { return BTFTypeBase::getSize() + 4; } + void completeType(BTFDebug &BDebug); + void emitType(MCStreamer &OS); +}; + +/// Handle data sections +class BTFKindDataSec : public BTFTypeBase { + AsmPrinter *Asm; + std::string Name; + std::vector<std::tuple<uint32_t, const MCSymbol *, uint32_t>> Vars; + +public: + BTFKindDataSec(AsmPrinter *AsmPrt, std::string SecName); + uint32_t getSize() { + return BTFTypeBase::getSize() + BTF::BTFDataSecVarSize * Vars.size(); + } + void addVar(uint32_t Id, const MCSymbol *Sym, uint32_t Size) { + Vars.push_back(std::make_tuple(Id, Sym, Size)); + } + std::string getName() { return Name; } + void completeType(BTFDebug &BDebug); + void emitType(MCStreamer &OS); +}; + /// String table. class BTFStringTable { /// String table size in bytes. @@ -189,6 +227,19 @@ struct BTFLineInfo { uint32_t ColumnNum; ///< the column number }; +/// Represent one offset relocation. +struct BTFOffsetReloc { + const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc + uint32_t TypeID; ///< Type ID + uint32_t OffsetNameOff; ///< The string to traverse types +}; + +/// Represent one extern relocation. +struct BTFExternReloc { + const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc + uint32_t ExternNameOff; ///< The extern variable name +}; + /// Collect and emit BTF information. class BTFDebug : public DebugHandlerBase { MCStreamer &OS; @@ -196,17 +247,26 @@ class BTFDebug : public DebugHandlerBase { bool LineInfoGenerated; uint32_t SecNameOff; uint32_t ArrayIndexTypeId; + bool MapDefNotCollected; BTFStringTable StringTable; std::vector<std::unique_ptr<BTFTypeBase>> TypeEntries; std::unordered_map<const DIType *, uint32_t> DIToIdMap; - std::unordered_map<uint32_t, std::vector<BTFFuncInfo>> FuncInfoTable; - std::unordered_map<uint32_t, std::vector<BTFLineInfo>> LineInfoTable; + std::map<uint32_t, std::vector<BTFFuncInfo>> FuncInfoTable; + std::map<uint32_t, std::vector<BTFLineInfo>> LineInfoTable; + std::map<uint32_t, std::vector<BTFOffsetReloc>> OffsetRelocTable; + std::map<uint32_t, std::vector<BTFExternReloc>> ExternRelocTable; StringMap<std::vector<std::string>> FileContent; + std::map<std::string, std::unique_ptr<BTFKindDataSec>> DataSecEntries; + std::vector<BTFTypeStruct *> StructTypes; + std::vector<BTFTypeArray *> ArrayTypes; + std::map<std::string, int64_t> AccessOffsets; + std::map<StringRef, std::pair<bool, std::vector<BTFTypeDerived *>>> + FixupDerivedTypes; /// Add types to TypeEntries. /// @{ /// Add types to TypeEntries and DIToIdMap. - void addType(std::unique_ptr<BTFTypeBase> TypeEntry, const DIType *Ty); + uint32_t addType(std::unique_ptr<BTFTypeBase> TypeEntry, const DIType *Ty); /// Add types to TypeEntries only and return type id. uint32_t addType(std::unique_ptr<BTFTypeBase> TypeEntry); /// @} @@ -214,17 +274,23 @@ class BTFDebug : public DebugHandlerBase { /// IR type visiting functions. /// @{ void visitTypeEntry(const DIType *Ty); - void visitBasicType(const DIBasicType *BTy); + void visitTypeEntry(const DIType *Ty, uint32_t &TypeId, bool CheckPointer, + bool SeenPointer); + void visitBasicType(const DIBasicType *BTy, uint32_t &TypeId); void visitSubroutineType( const DISubroutineType *STy, bool ForSubprog, const std::unordered_map<uint32_t, StringRef> &FuncArgNames, uint32_t &TypeId); - void visitFwdDeclType(const DICompositeType *CTy, bool IsUnion); - void visitCompositeType(const DICompositeType *CTy); - void visitStructType(const DICompositeType *STy, bool IsStruct); - void visitArrayType(const DICompositeType *ATy); - void visitEnumType(const DICompositeType *ETy); - void visitDerivedType(const DIDerivedType *DTy); + void visitFwdDeclType(const DICompositeType *CTy, bool IsUnion, + uint32_t &TypeId); + void visitCompositeType(const DICompositeType *CTy, uint32_t &TypeId); + void visitStructType(const DICompositeType *STy, bool IsStruct, + uint32_t &TypeId); + void visitArrayType(const DICompositeType *ATy, uint32_t &TypeId); + void visitEnumType(const DICompositeType *ETy, uint32_t &TypeId); + void visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId, + bool CheckPointer, bool SeenPointer); + void visitMapDefType(const DIType *Ty, uint32_t &TypeId); /// @} /// Get the file content for the subprogram. Certain lines of the file @@ -235,6 +301,23 @@ class BTFDebug : public DebugHandlerBase { void constructLineInfo(const DISubprogram *SP, MCSymbol *Label, uint32_t Line, uint32_t Column); + /// Generate types and variables for globals. + void processGlobals(bool ProcessingMapDef); + + /// Generate one offset relocation record. + void generateOffsetReloc(const MachineInstr *MI, const MCSymbol *ORSym, + DIType *RootTy, StringRef AccessPattern); + + /// Set the to-be-traversed Struct/Array Type based on TypeId. + void setTypeFromId(uint32_t TypeId, BTFTypeStruct **PrevStructType, + BTFTypeArray **PrevArrayType); + + /// Populating unprocessed struct type. + unsigned populateStructType(const DIType *Ty); + + /// Process LD_imm64 instructions. + void processLDimm64(const MachineInstr *MI); + /// Emit common header of .BTF and .BTF.ext sections. void emitCommonHeader(); @@ -254,6 +337,9 @@ protected: public: BTFDebug(AsmPrinter *AP); + /// + bool InstLower(const MachineInstr *MI, MCInst &OutMI); + /// Get the special array index type id. uint32_t getArrayIndexTypeId() { assert(ArrayIndexTypeId); diff --git a/contrib/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp b/contrib/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp index 9f80b762fe36..c845524ad657 100644 --- a/contrib/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp +++ b/contrib/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp @@ -1,9 +1,8 @@ //===- BPFDisassembler.cpp - Disassembler for BPF ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "MCTargetDesc/BPFMCTargetDesc.h" +#include "TargetInfo/BPFTargetInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" @@ -40,7 +40,7 @@ public: BPF_STX = 0x3, BPF_ALU = 0x4, BPF_JMP = 0x5, - BPF_RES = 0x6, + BPF_JMP32 = 0x6, BPF_ALU64 = 0x7 }; @@ -172,9 +172,10 @@ DecodeStatus BPFDisassembler::getInstruction(MCInst &Instr, uint64_t &Size, if (Result == MCDisassembler::Fail) return MCDisassembler::Fail; uint8_t InstClass = getInstClass(Insn); + uint8_t InstMode = getInstMode(Insn); if ((InstClass == BPF_LDX || InstClass == BPF_STX) && getInstSize(Insn) != BPF_DW && - getInstMode(Insn) == BPF_MEM && + (InstMode == BPF_MEM || InstMode == BPF_XADD) && STI.getFeatureBits()[BPF::ALU32]) Result = decodeInstruction(DecoderTableBPFALU3264, Instr, Insn, Address, this, STI); diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp index 1822d8688fa2..ba35a175b9a7 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp @@ -1,9 +1,8 @@ //===-- BPFAsmBackend.cpp - BPF Assembler Backend -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -73,12 +72,12 @@ void BPFAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, bool IsResolved, const MCSubtargetInfo *STI) const { if (Fixup.getKind() == FK_SecRel_4 || Fixup.getKind() == FK_SecRel_8) { - if (Value) { - MCContext &Ctx = Asm.getContext(); - Ctx.reportError(Fixup.getLoc(), - "Unsupported relocation: try to compile with -O2 or above, " - "or check your static variable usage"); - } + // The Value is 0 for global variables, and the in-section offset + // for static variables. Write to the immediate field of the inst. + assert(Value <= UINT32_MAX); + support::endian::write<uint32_t>(&Data[Fixup.getOffset() + 4], + static_cast<uint32_t>(Value), + Endian); } else if (Fixup.getKind() == FK_Data_4) { support::endian::write<uint32_t>(&Data[Fixup.getOffset()], Value, Endian); } else if (Fixup.getKind() == FK_Data_8) { diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp index 32e79d0f527e..057bbf5c3b06 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp @@ -1,9 +1,8 @@ //===-- BPFELFObjectWriter.cpp - BPF ELF Writer ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -51,21 +50,33 @@ unsigned BPFELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target, case FK_Data_8: return ELF::R_BPF_64_64; case FK_Data_4: - // .BTF.ext generates FK_Data_4 relocations for - // insn offset by creating temporary labels. - // The insn offset is within the code section and - // already been fulfilled by applyFixup(). No - // further relocation is needed. if (const MCSymbolRefExpr *A = Target.getSymA()) { - if (A->getSymbol().isTemporary()) { - MCSection &Section = A->getSymbol().getSection(); + const MCSymbol &Sym = A->getSymbol(); + + if (Sym.isDefined()) { + MCSection &Section = Sym.getSection(); const MCSectionELF *SectionELF = dyn_cast<MCSectionELF>(&Section); assert(SectionELF && "Null section for reloc symbol"); - // The reloc symbol should be in text section. unsigned Flags = SectionELF->getFlags(); - if ((Flags & ELF::SHF_ALLOC) && (Flags & ELF::SHF_EXECINSTR)) - return ELF::R_BPF_NONE; + + if (Sym.isTemporary()) { + // .BTF.ext generates FK_Data_4 relocations for + // insn offset by creating temporary labels. + // The insn offset is within the code section and + // already been fulfilled by applyFixup(). No + // further relocation is needed. + // The reloc symbol should be in text section. + if ((Flags & ELF::SHF_ALLOC) && (Flags & ELF::SHF_EXECINSTR)) + return ELF::R_BPF_NONE; + } else { + // .BTF generates FK_Data_4 relocations for variable + // offset in DataSec kind. Similar to the above .BTF.ext + // insn offset, no further relocation is needed. + // The reloc symbol should be in data section. + if ((Flags & ELF::SHF_ALLOC) && (Flags & ELF::SHF_WRITE)) + return ELF::R_BPF_NONE; + } } } return ELF::R_BPF_64_32; diff --git a/contrib/llvm/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp index 20627da38817..079202994c8d 100644 --- a/contrib/llvm/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp @@ -1,9 +1,8 @@ //===-- BPFInstPrinter.cpp - Convert BPF MCInst to asm syntax -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -11,7 +10,7 @@ // //===----------------------------------------------------------------------===// -#include "BPFInstPrinter.h" +#include "MCTargetDesc/BPFInstPrinter.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" diff --git a/contrib/llvm/lib/Target/BPF/InstPrinter/BPFInstPrinter.h b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.h index bb0b0d71da53..8c9a0bc94cff 100644 --- a/contrib/llvm/lib/Target/BPF/InstPrinter/BPFInstPrinter.h +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.h @@ -1,9 +1,8 @@ //===-- BPFInstPrinter.h - Convert BPF MCInst to asm syntax -------*- C++ -*--// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -11,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_TARGET_BPF_INSTPRINTER_BPFINSTPRINTER_H -#define LLVM_LIB_TARGET_BPF_INSTPRINTER_BPFINSTPRINTER_H +#ifndef LLVM_LIB_TARGET_BPF_MCTARGETDESC_BPFINSTPRINTER_H +#define LLVM_LIB_TARGET_BPF_MCTARGETDESC_BPFINSTPRINTER_H #include "llvm/MC/MCInstPrinter.h" diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h index af3ad5315253..04a6a87cebc9 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h @@ -1,9 +1,8 @@ //===-- BPFMCAsmInfo.h - BPF asm properties -------------------*- C++ -*--====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp index 437f658caf6e..f9abe76c976b 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp @@ -1,9 +1,8 @@ //===-- BPFMCCodeEmitter.cpp - Convert BPF code to machine code -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -64,9 +63,10 @@ public: const MCSubtargetInfo &STI) const override; private: - uint64_t computeAvailableFeatures(const FeatureBitset &FB) const; - void verifyInstructionPredicates(const MCInst &MI, - uint64_t AvailableFeatures) const; + FeatureBitset computeAvailableFeatures(const FeatureBitset &FB) const; + void + verifyInstructionPredicates(const MCInst &MI, + const FeatureBitset &AvailableFeatures) const; }; } // end anonymous namespace diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp index 834b57527882..fa27b335f3a1 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp @@ -1,9 +1,8 @@ //===-- BPFMCTargetDesc.cpp - BPF Target Descriptions ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,9 +11,9 @@ //===----------------------------------------------------------------------===// #include "MCTargetDesc/BPFMCTargetDesc.h" -#include "BPF.h" -#include "InstPrinter/BPFInstPrinter.h" +#include "MCTargetDesc/BPFInstPrinter.h" #include "MCTargetDesc/BPFMCAsmInfo.h" +#include "TargetInfo/BPFTargetInfo.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h index 6d2f0a1601e6..1a391321f60d 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h @@ -1,9 +1,8 @@ //===-- BPFMCTargetDesc.h - BPF Target Descriptions -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,10 +33,6 @@ class Triple; class raw_ostream; class raw_pwrite_stream; -Target &getTheBPFleTarget(); -Target &getTheBPFbeTarget(); -Target &getTheBPFTarget(); - MCCodeEmitter *createBPFMCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI, MCContext &Ctx); diff --git a/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp b/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp index 1f7b8a04d589..5dfa915034ba 100644 --- a/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp +++ b/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp @@ -1,30 +1,28 @@ //===-- BPFTargetInfo.cpp - BPF Target Implementation ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include "BPF.h" +#include "TargetInfo/BPFTargetInfo.h" #include "llvm/Support/TargetRegistry.h" + using namespace llvm; -namespace llvm { -Target &getTheBPFleTarget() { +Target &llvm::getTheBPFleTarget() { static Target TheBPFleTarget; return TheBPFleTarget; } -Target &getTheBPFbeTarget() { +Target &llvm::getTheBPFbeTarget() { static Target TheBPFbeTarget; return TheBPFbeTarget; } -Target &getTheBPFTarget() { +Target &llvm::getTheBPFTarget() { static Target TheBPFTarget; return TheBPFTarget; } -} // namespace llvm extern "C" void LLVMInitializeBPFTargetInfo() { TargetRegistry::RegisterTarget(getTheBPFTarget(), "bpf", "BPF (host endian)", diff --git a/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.h b/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.h new file mode 100644 index 000000000000..150526c1a9db --- /dev/null +++ b/contrib/llvm/lib/Target/BPF/TargetInfo/BPFTargetInfo.h @@ -0,0 +1,22 @@ +//===-- BPFTargetInfo.h - BPF Target Implementation -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_TARGETINFO_BPFTARGETINFO_H +#define LLVM_LIB_TARGET_BPF_TARGETINFO_BPFTARGETINFO_H + +namespace llvm { + +class Target; + +Target &getTheBPFleTarget(); +Target &getTheBPFbeTarget(); +Target &getTheBPFTarget(); + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_BPF_TARGETINFO_BPFTARGETINFO_H |