diff options
Diffstat (limited to 'include/llvm/CodeGen/GlobalISel')
25 files changed, 1351 insertions, 332 deletions
diff --git a/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/include/llvm/CodeGen/GlobalISel/CSEInfo.h index ce2d285a99e53..5a44e67992ad7 100644 --- a/include/llvm/CodeGen/GlobalISel/CSEInfo.h +++ b/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/CSEInfo.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H #include "llvm/ADT/FoldingSet.h" +#include "llvm/CodeGen/CSEConfigBase.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelWorkList.h" #include "llvm/CodeGen/GlobalISel/Utils.h" @@ -37,25 +37,27 @@ public: void Profile(FoldingSetNodeID &ID); }; -// Class representing some configuration that can be done during CSE analysis. -// Currently it only supports shouldCSE method that each pass can set. -class CSEConfig { +// A CSE config for fully optimized builds. +class CSEConfigFull : public CSEConfigBase { public: - virtual ~CSEConfig() = default; - // Hook for defining which Generic instructions should be CSEd. - // GISelCSEInfo currently only calls this hook when dealing with generic - // opcodes. - virtual bool shouldCSEOpc(unsigned Opc); + virtual ~CSEConfigFull() = default; + virtual bool shouldCSEOpc(unsigned Opc) override; }; -// TODO: Find a better place for this. // Commonly used for O0 config. -class CSEConfigConstantOnly : public CSEConfig { +class CSEConfigConstantOnly : public CSEConfigBase { public: virtual ~CSEConfigConstantOnly() = default; virtual bool shouldCSEOpc(unsigned Opc) override; }; +// Returns the standard expected CSEConfig for the given optimization level. +// We have this logic here so targets can make use of it from their derived +// TargetPassConfig, but can't put this logic into TargetPassConfig directly +// because the CodeGen library can't depend on GlobalISel. +std::unique_ptr<CSEConfigBase> +getStandardCSEConfigForOpt(CodeGenOpt::Level Level); + /// The CSE Analysis object. /// This installs itself as a delegate to the MachineFunction to track /// new instructions as well as deletions. It however will not be able to @@ -74,7 +76,7 @@ class GISelCSEInfo : public GISelChangeObserver { FoldingSet<UniqueMachineInstr> CSEMap; MachineRegisterInfo *MRI = nullptr; MachineFunction *MF = nullptr; - std::unique_ptr<CSEConfig> CSEOpt; + std::unique_ptr<CSEConfigBase> CSEOpt; /// Keep a cache of UniqueInstrs for each MachineInstr. In GISel, /// often instructions are mutated (while their ID has completely changed). /// Whenever mutation happens, invalidate the UniqueMachineInstr for the @@ -139,7 +141,9 @@ public: void releaseMemory(); - void setCSEConfig(std::unique_ptr<CSEConfig> Opt) { CSEOpt = std::move(Opt); } + void setCSEConfig(std::unique_ptr<CSEConfigBase> Opt) { + CSEOpt = std::move(Opt); + } bool shouldCSE(unsigned Opc) const; @@ -199,11 +203,12 @@ class GISelCSEAnalysisWrapper { bool AlreadyComputed = false; public: - /// Takes a CSEConfig object that defines what opcodes get CSEd. + /// Takes a CSEConfigBase object that defines what opcodes get CSEd. /// If CSEConfig is already set, and the CSE Analysis has been preserved, /// it will not use the new CSEOpt(use Recompute to force using the new /// CSEOpt). - GISelCSEInfo &get(std::unique_ptr<CSEConfig> CSEOpt, bool ReCompute = false); + GISelCSEInfo &get(std::unique_ptr<CSEConfigBase> CSEOpt, + bool ReCompute = false); void setMF(MachineFunction &MFunc) { MF = &MFunc; } void setComputed(bool Computed) { AlreadyComputed = Computed; } void releaseMemory() { Info.releaseMemory(); } diff --git a/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h index a8fb736ebbb5d..4f95335db74b2 100644 --- a/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.h --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h index ab498e8f070b0..d717121ad78ec 100644 --- a/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/CallLowering.h - Call lowering ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -16,6 +15,7 @@ #define LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallSite.h" @@ -27,6 +27,7 @@ namespace llvm { +class CCState; class DataLayout; class Function; class MachineIRBuilder; @@ -43,14 +44,19 @@ class CallLowering { virtual void anchor(); public: struct ArgInfo { - unsigned Reg; + SmallVector<Register, 4> Regs; Type *Ty; ISD::ArgFlagsTy Flags; bool IsFixed; - ArgInfo(unsigned Reg, Type *Ty, ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, - bool IsFixed = true) - : Reg(Reg), Ty(Ty), Flags(Flags), IsFixed(IsFixed) {} + ArgInfo(ArrayRef<Register> Regs, Type *Ty, + ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, bool IsFixed = true) + : Regs(Regs.begin(), Regs.end()), Ty(Ty), Flags(Flags), + IsFixed(IsFixed) { + // FIXME: We should have just one way of saying "no register". + assert((Ty->isVoidTy() == (Regs.empty() || Regs[0] == 0)) && + "only void types should have no register"); + } }; /// Argument handling is mostly uniform between the four places that @@ -66,24 +72,28 @@ public: virtual ~ValueHandler() = default; + /// Returns true if the handler is dealing with formal arguments, + /// not with return values etc. + virtual bool isArgumentHandler() const { return false; } + /// Materialize a VReg containing the address of the specified /// stack-based object. This is either based on a FrameIndex or /// direct SP manipulation, depending on the context. \p MPO /// should be initialized to an appropriate description of the /// address created. - virtual unsigned getStackAddress(uint64_t Size, int64_t Offset, + virtual Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) = 0; /// The specified value has been assigned to a physical register, /// handle the appropriate COPY (either to or from) and mark any /// relevant uses/defines as needed. - virtual void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + virtual void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) = 0; /// The specified value has been assigned to a stack /// location. Load or store it there, with appropriate extension /// if necessary. - virtual void assignValueToAddress(unsigned ValVReg, unsigned Addr, + virtual void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) = 0; @@ -98,7 +108,7 @@ public: llvm_unreachable("Custom values not supported"); } - unsigned extendRegister(unsigned ValReg, CCValAssign &VA); + Register extendRegister(Register ValReg, CCValAssign &VA); virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, const ArgInfo &Info, @@ -130,39 +140,83 @@ protected: void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const; + /// Generate instructions for packing \p SrcRegs into one big register + /// corresponding to the aggregate type \p PackedTy. + /// + /// \param SrcRegs should contain one virtual register for each base type in + /// \p PackedTy, as returned by computeValueLLTs. + /// + /// \return The packed register. + Register packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy, + MachineIRBuilder &MIRBuilder) const; + + /// Generate instructions for unpacking \p SrcReg into the \p DstRegs + /// corresponding to the aggregate type \p PackedTy. + /// + /// \param DstRegs should contain one virtual register for each base type in + /// \p PackedTy, as returned by computeValueLLTs. + void unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg, Type *PackedTy, + MachineIRBuilder &MIRBuilder) const; + /// Invoke Handler::assignArg on each of the given \p Args and then use /// \p Callback to move them to the assigned locations. /// /// \return True if everything has succeeded, false otherwise. bool handleAssignments(MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args, ValueHandler &Handler) const; - + bool handleAssignments(CCState &CCState, + SmallVectorImpl<CCValAssign> &ArgLocs, + MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args, + ValueHandler &Handler) const; public: CallLowering(const TargetLowering *TLI) : TLI(TLI) {} virtual ~CallLowering() = default; + /// \return true if the target is capable of handling swifterror values that + /// have been promoted to a specified register. The extended versions of + /// lowerReturn and lowerCall should be implemented. + virtual bool supportSwiftError() const { + return false; + } + /// This hook must be implemented to lower outgoing return values, described /// by \p Val, into the specified virtual registers \p VRegs. /// This hook is used by GlobalISel. /// + /// \p SwiftErrorVReg is non-zero if the function has a swifterror parameter + /// that needs to be implicitly returned. + /// /// \return True if the lowering succeeds, false otherwise. virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, - ArrayRef<unsigned> VRegs) const { + ArrayRef<Register> VRegs, + Register SwiftErrorVReg) const { + if (!supportSwiftError()) { + assert(SwiftErrorVReg == 0 && "attempt to use unsupported swifterror"); + return lowerReturn(MIRBuilder, Val, VRegs); + } + return false; + } + + /// This hook behaves as the extended lowerReturn function, but for targets + /// that do not support swifterror value promotion. + virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, + ArrayRef<Register> VRegs) const { return false; } /// This hook must be implemented to lower the incoming (formal) - /// arguments, described by \p Args, for GlobalISel. Each argument - /// must end up in the related virtual register described by VRegs. - /// In other words, the first argument should end up in VRegs[0], - /// the second in VRegs[1], and so on. + /// arguments, described by \p VRegs, for GlobalISel. Each argument + /// must end up in the related virtual registers described by \p VRegs. + /// In other words, the first argument should end up in \c VRegs[0], + /// the second in \c VRegs[1], and so on. For each argument, there will be one + /// register for each non-aggregate type, as returned by \c computeValueLLTs. /// \p MIRBuilder is set to the proper insertion for the argument /// lowering. /// /// \return True if the lowering succeeded, false otherwise. virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, - ArrayRef<unsigned> VRegs) const { + ArrayRef<ArrayRef<Register>> VRegs) const { return false; } @@ -174,20 +228,31 @@ public: /// \p Callee is the destination of the call. It should be either a register, /// globaladdress, or externalsymbol. /// - /// \p ResTy is the type returned by the function - /// - /// \p ResReg is the generic virtual register that the returned - /// value should be lowered into. + /// \p OrigRet is a descriptor for the return type of the function. /// - /// \p ArgTys is a list of the types each member of \p ArgRegs has; used by - /// the target to decide which register/stack slot should be allocated. + /// \p OrigArgs is a list of descriptors of the arguments passed to the + /// function. /// - /// \p ArgRegs is a list of virtual registers containing each argument that - /// needs to be passed. + /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout + /// parameter, and contains the vreg that the swifterror should be copied into + /// after the call. /// /// \return true if the lowering succeeded, false otherwise. virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, const MachineOperand &Callee, const ArgInfo &OrigRet, + ArrayRef<ArgInfo> OrigArgs, + Register SwiftErrorVReg) const { + if (!supportSwiftError()) { + assert(SwiftErrorVReg == 0 && "trying to use unsupported swifterror"); + return lowerCall(MIRBuilder, CallConv, Callee, OrigRet, OrigArgs); + } + return false; + } + + /// This hook behaves as the extended lowerCall function, but for targets that + /// do not support swifterror value promotion. + virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, + const MachineOperand &Callee, const ArgInfo &OrigRet, ArrayRef<ArgInfo> OrigArgs) const { return false; } @@ -197,11 +262,18 @@ public: /// /// \p CI is the call/invoke instruction. /// - /// \p ResReg is a register where the call's return value should be stored (or - /// 0 if there is no return value). + /// \p ResRegs are the registers where the call's return value should be + /// stored (or 0 if there is no return value). There will be one register for + /// each non-aggregate type, as returned by \c computeValueLLTs. + /// + /// \p ArgRegs is a list of lists of virtual registers containing each + /// argument that needs to be passed (argument \c i should be placed in \c + /// ArgRegs[i]). For each argument, there will be one register for each + /// non-aggregate type, as returned by \c computeValueLLTs. /// - /// \p ArgRegs is a list of virtual registers containing each argument that - /// needs to be passed. + /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout + /// parameter, and contains the vreg that the swifterror should be copied into + /// after the call. /// /// \p GetCalleeReg is a callback to materialize a register for the callee if /// the target determines it cannot jump to the destination based purely on \p @@ -210,7 +282,8 @@ public: /// /// \return true if the lowering succeeded, false otherwise. bool lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS, - unsigned ResReg, ArrayRef<unsigned> ArgRegs, + ArrayRef<Register> ResRegs, + ArrayRef<ArrayRef<Register>> ArgRegs, Register SwiftErrorVReg, std::function<unsigned()> GetCalleeReg) const; }; diff --git a/include/llvm/CodeGen/GlobalISel/Combiner.h b/include/llvm/CodeGen/GlobalISel/Combiner.h index b097c78177624..efe8bdf93664a 100644 --- a/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -1,9 +1,8 @@ -//== ----- llvm/CodeGen/GlobalISel/Combiner.h --------------------- == // +//== ----- llvm/CodeGen/GlobalISel/Combiner.h -------------------*- C++ -*-== // // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 6e9ac01c1ee29..0c50c9c5e0cf2 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/CombinerHelper.h --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===--------------------------------------------------------------------===// // @@ -18,6 +17,9 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H #define LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H +#include "llvm/CodeGen/LowLevelType.h" +#include "llvm/CodeGen/Register.h" + namespace llvm { class GISelChangeObserver; @@ -26,6 +28,12 @@ class MachineRegisterInfo; class MachineInstr; class MachineOperand; +struct PreferredTuple { + LLT Ty; // The result type of the extend. + unsigned ExtendOpcode; // G_ANYEXT/G_SEXT/G_ZEXT + MachineInstr *MI; +}; + class CombinerHelper { MachineIRBuilder &Builder; MachineRegisterInfo &MRI; @@ -35,20 +43,27 @@ public: CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B); /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes - void replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, unsigned ToReg) const; + void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const; /// Replace a single register operand with a new register and inform the /// observer of the changes. void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, - unsigned ToReg) const; + Register ToReg) const; /// If \p MI is COPY, try to combine it. /// Returns true if MI changed. bool tryCombineCopy(MachineInstr &MI); + bool matchCombineCopy(MachineInstr &MI); + void applyCombineCopy(MachineInstr &MI); /// If \p MI is extend that consumes the result of a load, try to combine it. /// Returns true if MI changed. bool tryCombineExtendingLoads(MachineInstr &MI); + bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); + void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); + + bool matchCombineBr(MachineInstr &MI); + bool tryCombineBr(MachineInstr &MI); /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index d21aa3f725d92..3b09a8e2b479f 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/CombinerInfo.h ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h index 220a571b21dbc..e817d9b4550e5 100644 --- a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h index c8e8a7a5a7cb2..e5691cb35174a 100644 --- a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h +++ b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h @@ -1,9 +1,8 @@ -//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h ------------------===// +//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,10 +33,17 @@ public: /// An instruction is about to be erased. virtual void erasingInstr(MachineInstr &MI) = 0; - /// An instruction was created and inserted into the function. + + /// An instruction has been created and inserted into the function. + /// Note that the instruction might not be a fully fledged instruction at this + /// point and won't be if the MachineFunction::Delegate is calling it. This is + /// because the delegate only sees the construction of the MachineInstr before + /// operands have been added. virtual void createdInstr(MachineInstr &MI) = 0; + /// This instruction is about to be mutated in some way. virtual void changingInstr(MachineInstr &MI) = 0; + /// This instruction was mutated in some way. virtual void changedInstr(MachineInstr &MI) = 0; diff --git a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index 1571841a208d0..b0bb519283b10 100644 --- a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -1,9 +1,8 @@ //===- GISelWorkList.h - Worklist for GISel passes ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -33,23 +32,61 @@ class GISelWorkList { SmallVector<MachineInstr *, N> Worklist; DenseMap<MachineInstr *, unsigned> WorklistMap; +#ifndef NDEBUG + bool Finalized = true; +#endif + public: - GISelWorkList() {} + GISelWorkList() : WorklistMap(N) {} bool empty() const { return WorklistMap.empty(); } unsigned size() const { return WorklistMap.size(); } + // Since we don't know ahead of time how many instructions we're going to add + // to the worklist, and migrating densemap's elements is quite expensive + // everytime we resize, only insert to the smallvector (typically during the + // initial phase of populating lists). Before the worklist can be used, + // finalize should be called. Also assert with NDEBUG if list is ever used + // without finalizing. Note that unlike insert, we won't check for duplicates + // - so the ideal place to use this is during the initial prepopulating phase + // of most passes. + void deferred_insert(MachineInstr *I) { + Worklist.push_back(I); +#ifndef NDEBUG + Finalized = false; +#endif + } + + // This should only be called when using deferred_insert. + // This asserts that the WorklistMap is empty, and then + // inserts all the elements in the Worklist into the map. + // It also asserts if there are any duplicate elements found. + void finalize() { + assert(WorklistMap.empty() && "Expecting empty worklistmap"); + if (Worklist.size() > N) + WorklistMap.reserve(Worklist.size()); + for (unsigned i = 0; i < Worklist.size(); ++i) + if (!WorklistMap.try_emplace(Worklist[i], i).second) + llvm_unreachable("Duplicate elements in the list"); +#ifndef NDEBUG + Finalized = true; +#endif + } + /// Add the specified instruction to the worklist if it isn't already in it. void insert(MachineInstr *I) { + assert(Finalized && "GISelWorkList used without finalizing"); if (WorklistMap.try_emplace(I, Worklist.size()).second) Worklist.push_back(I); } /// Remove I from the worklist if it exists. void remove(const MachineInstr *I) { + assert((Finalized || WorklistMap.empty()) && "Neither finalized nor empty"); auto It = WorklistMap.find(I); - if (It == WorklistMap.end()) return; // Not in worklist. + if (It == WorklistMap.end()) + return; // Not in worklist. // Don't bother moving everything down, just null out the slot. Worklist[It->second] = nullptr; @@ -63,6 +100,7 @@ public: } MachineInstr *pop_back_val() { + assert(Finalized && "GISelWorkList used without finalizing"); MachineInstr *I; do { I = Worklist.pop_back_val(); diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h index d1770bf6e4cea..8654ba83f08d3 100644 --- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/IRTranslator.h - IRTranslator ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -23,7 +22,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Types.h" +#include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/SwitchLoweringUtils.h" #include "llvm/IR/Intrinsics.h" #include "llvm/Support/Allocator.h" #include <memory> @@ -37,6 +38,7 @@ class CallInst; class CallLowering; class Constant; class DataLayout; +class FunctionLoweringInfo; class Instruction; class MachineBasicBlock; class MachineFunction; @@ -69,7 +71,7 @@ private: public: ValueToVRegInfo() = default; - using VRegListT = SmallVector<unsigned, 1>; + using VRegListT = SmallVector<Register, 1>; using OffsetListT = SmallVector<uint64_t, 1>; using const_vreg_iterator = @@ -164,6 +166,8 @@ private: /// this function. DenseMap<const AllocaInst *, int> FrameIndices; + SwiftErrorValueTracking SwiftError; + /// \name Methods for translating form LLVM IR to MachineInstr. /// \see ::translate for general information on the translate methods. /// @{ @@ -196,7 +200,7 @@ private: /// the function. /// /// \return true if the materialization succeeded. - bool translate(const Constant &C, unsigned Reg); + bool translate(const Constant &C, Register Reg); /// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is /// emitted. @@ -212,24 +216,27 @@ private: bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned ID); - void getStackGuard(unsigned DstReg, MachineIRBuilder &MIRBuilder); + void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder); bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder); + /// Helper function for translateSimpleIntrinsic. + /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a + /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns + /// Intrinsic::not_intrinsic. + unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID); + + /// Translates the intrinsics defined in getSimpleIntrinsicOpcode. + /// \return true if the translation succeeded. + bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID, + MachineIRBuilder &MIRBuilder); + bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); bool translateInlineAsm(const CallInst &CI, MachineIRBuilder &MIRBuilder); - // FIXME: temporary function to expose previous interface to call lowering - // until it is refactored. - /// Combines all component registers of \p V into a single scalar with size - /// "max(Offsets) + last size". - unsigned packRegs(const Value &V, MachineIRBuilder &MIRBuilder); - - void unpackRegs(const Value &V, unsigned Src, MachineIRBuilder &MIRBuilder); - /// Returns true if the value should be split into multiple LLTs. /// If \p Offsets is given then the split type's offsets will be stored in it. /// If \p Offsets is not empty it will be cleared first. @@ -242,6 +249,8 @@ private: bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); + bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); + bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); /// Translate one of LLVM's cast instructions into MachineInstrs, with the @@ -278,7 +287,42 @@ private: /// \pre \p U is a branch instruction. bool translateBr(const User &U, MachineIRBuilder &MIRBuilder); + // Begin switch lowering functions. + bool emitJumpTableHeader(SwitchCG::JumpTable &JT, + SwitchCG::JumpTableHeader &JTH, + MachineBasicBlock *HeaderBB); + void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB); + + void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB, + MachineIRBuilder &MIB); + + bool lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W, + MachineBasicBlock *SwitchMBB, + MachineBasicBlock *CurMBB, + MachineBasicBlock *DefaultMBB, + MachineIRBuilder &MIB, + MachineFunction::iterator BBI, + BranchProbability UnhandledProbs, + SwitchCG::CaseClusterIt I, + MachineBasicBlock *Fallthrough, + bool FallthroughUnreachable); + + bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, + Value *Cond, + MachineBasicBlock *Fallthrough, + bool FallthroughUnreachable, + BranchProbability UnhandledProbs, + MachineBasicBlock *CurMBB, + MachineIRBuilder &MIB, + MachineBasicBlock *SwitchMBB); + + bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond, + MachineBasicBlock *SwitchMBB, + MachineBasicBlock *DefaultMBB, + MachineIRBuilder &MIB); + bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder); + // End switch lowering section. bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder); @@ -404,6 +448,7 @@ private: bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder); bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder); + bool translateFence(const User &U, MachineIRBuilder &MIRBuilder); // Stubs to keep the compiler happy while we implement the rest of the // translation. @@ -419,9 +464,6 @@ private: bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) { return false; } - bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) { - return false; - } bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) { return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder); } @@ -466,19 +508,50 @@ private: /// Current optimization remark emitter. Used to report failures. std::unique_ptr<OptimizationRemarkEmitter> ORE; + FunctionLoweringInfo FuncInfo; + + // True when either the Target Machine specifies no optimizations or the + // function has the optnone attribute. + bool EnableOpts = false; + + /// Switch analysis and optimization. + class GISelSwitchLowering : public SwitchCG::SwitchLowering { + public: + GISelSwitchLowering(IRTranslator *irt, FunctionLoweringInfo &funcinfo) + : SwitchLowering(funcinfo), IRT(irt) { + assert(irt && "irt is null!"); + } + + virtual void addSuccessorWithProb( + MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob = BranchProbability::getUnknown()) override { + IRT->addSuccessorWithProb(Src, Dst, Prob); + } + + virtual ~GISelSwitchLowering() = default; + + private: + IRTranslator *IRT; + }; + + std::unique_ptr<GISelSwitchLowering> SL; + // * Insert all the code needed to materialize the constants // at the proper place. E.g., Entry block or dominator block // of each constant depending on how fancy we want to be. // * Clear the different maps. void finalizeFunction(); + // Handle emitting jump tables for each basic block. + void finalizeBasicBlock(); + /// Get the VRegs that represent \p Val. /// Non-aggregate types have just one corresponding VReg and the list can be /// used as a single "unsigned". Aggregates get flattened. If such VRegs do /// not exist, they are created. - ArrayRef<unsigned> getOrCreateVRegs(const Value &Val); + ArrayRef<Register> getOrCreateVRegs(const Value &Val); - unsigned getOrCreateVReg(const Value &Val) { + Register getOrCreateVReg(const Value &Val) { auto Regs = getOrCreateVRegs(Val); if (Regs.empty()) return 0; @@ -522,6 +595,14 @@ private: return SmallVector<MachineBasicBlock *, 4>(1, &getMBB(*Edge.first)); } + /// Return branch probability calculated by BranchProbabilityInfo for IR + /// blocks. + BranchProbability getEdgeProbability(const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const; + + void addSuccessorWithProb(MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob); + public: // Ctor, nothing fancy. IRTranslator(); diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelect.h b/include/llvm/CodeGen/GlobalISel/InstructionSelect.h index 01521c46ab6aa..1af46e0a9e76b 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelect.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelect.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/GlobalISel/InstructionSelect.h -----------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file This file describes the interface of the MachineFunctionPass diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 471def7f45a3d..e9b93be76754f 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/InstructionSelector.h ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -139,6 +138,16 @@ enum { /// - MMOIdx - MMO index /// - Size - The size in bytes of the memory access GIM_CheckMemorySizeEqualTo, + + /// Check the address space of the memory access for the given machine memory + /// operand. + /// - InsnID - Instruction ID + /// - MMOIdx - MMO index + /// - NumAddrSpace - Number of valid address spaces + /// - AddrSpaceN - An allowed space of the memory access + /// - AddrSpaceN+1 ... + GIM_CheckMemoryAddressSpace, + /// Check the size of the memory access for the given machine memory operand /// against the size of an operand. /// - InsnID - Instruction ID diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index 2003a79f6b206..e8ee4af0cb0b5 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -371,6 +370,45 @@ bool InstructionSelector::executeMatchTable( return false; break; } + case GIM_CheckMemoryAddressSpace: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t MMOIdx = MatchTable[CurrentIdx++]; + // This accepts a list of possible address spaces. + const int NumAddrSpace = MatchTable[CurrentIdx++]; + + if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + + // Need to still jump to the end of the list of address spaces if we find + // a match earlier. + const uint64_t LastIdx = CurrentIdx + NumAddrSpace; + + const MachineMemOperand *MMO + = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); + const unsigned MMOAddrSpace = MMO->getAddrSpace(); + + bool Success = false; + for (int I = 0; I != NumAddrSpace; ++I) { + unsigned AddrSpace = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE( + TgtInstructionSelector::getName(), + dbgs() << "addrspace(" << MMOAddrSpace << ") vs " + << AddrSpace << '\n'); + + if (AddrSpace == MMOAddrSpace) { + Success = true; + break; + } + } + + CurrentIdx = LastIdx; + if (!Success && handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckMemorySizeEqualTo: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t MMOIdx = MatchTable[CurrentIdx++]; @@ -438,15 +476,15 @@ bool InstructionSelector::executeMatchTable( unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && - MMO->getSize() * 8 != Size) { + MMO->getSizeInBits() != Size) { if (handleReject() == RejectAndGiveUp) return false; } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && - MMO->getSize() * 8 >= Size) { + MMO->getSizeInBits() >= Size) { if (handleReject() == RejectAndGiveUp) return false; } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && - MMO->getSize() * 8 <= Size) + MMO->getSizeInBits() <= Size) if (handleReject() == RejectAndGiveUp) return false; @@ -479,17 +517,19 @@ bool InstructionSelector::executeMatchTable( << InsnID << "]->getOperand(" << OpIdx << "), SizeInBits=" << SizeInBits << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + const LLT Ty = MRI.getType(MO.getReg()); + // iPTR must be looked up in the target. if (SizeInBits == 0) { MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); - SizeInBits = MF->getDataLayout().getPointerSizeInBits(0); + const unsigned AddrSpace = Ty.getAddressSpace(); + SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); } assert(SizeInBits != 0 && "Pointer size must be known"); - MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); if (MO.isReg()) { - const LLT &Ty = MRI.getType(MO.getReg()); if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) if (handleReject() == RejectAndGiveUp) return false; diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 20bec76501796..a22778b8848c7 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h -----*- C++ -*-// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file contains some helper functions which try to cleanup artifacts @@ -29,6 +28,18 @@ class LegalizationArtifactCombiner { MachineRegisterInfo &MRI; const LegalizerInfo &LI; + static bool isArtifactCast(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_TRUNC: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_ANYEXT: + return true; + default: + return false; + } + } + public: LegalizationArtifactCombiner(MachineIRBuilder &B, MachineRegisterInfo &MRI, const LegalizerInfo &LI) @@ -40,11 +51,11 @@ public: return false; Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // aext(trunc x) - > aext/copy/trunc x - unsigned TruncSrc; + Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); Builder.buildAnyExtOrTrunc(DstReg, TruncSrc); @@ -53,7 +64,7 @@ public: } // aext([asz]ext x) -> [asz]ext x - unsigned ExtSrc; + Register ExtSrc; MachineInstr *ExtMI; if (mi_match(SrcReg, MRI, m_all_of(m_MInstr(ExtMI), m_any_of(m_GAnyExt(m_Reg(ExtSrc)), @@ -63,6 +74,20 @@ public: markInstAndDefDead(MI, *ExtMI, DeadInsts); return true; } + + // Try to fold aext(g_constant) when the larger constant type is legal. + // Can't use MIPattern because we don't have a specific constant in mind. + auto *SrcMI = MRI.getVRegDef(SrcReg); + if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { + const LLT &DstTy = MRI.getType(DstReg); + if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) { + auto &CstVal = SrcMI->getOperand(1); + Builder.buildConstant( + DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits())); + markInstAndDefDead(MI, *SrcMI, DeadInsts); + return true; + } + } return tryFoldImplicitDef(MI, DeadInsts); } @@ -73,25 +98,39 @@ public: return false; Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // zext(trunc x) - > and (aext/copy/trunc x), mask - unsigned TruncSrc; + Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) || - isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + isConstantUnsupported(DstTy)) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); LLT SrcTy = MRI.getType(SrcReg); - APInt Mask = APInt::getAllOnesValue(SrcTy.getSizeInBits()); + APInt Mask = APInt::getAllOnesValue(SrcTy.getScalarSizeInBits()); auto MIBMask = Builder.buildConstant(DstTy, Mask.getZExtValue()); Builder.buildAnd(DstReg, Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBMask); markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } + + // Try to fold zext(g_constant) when the larger constant type is legal. + // Can't use MIPattern because we don't have a specific constant in mind. + auto *SrcMI = MRI.getVRegDef(SrcReg); + if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { + const LLT &DstTy = MRI.getType(DstReg); + if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) { + auto &CstVal = SrcMI->getOperand(1); + Builder.buildConstant( + DstReg, CstVal.getCImm()->getValue().zext(DstTy.getSizeInBits())); + markInstAndDefDead(MI, *SrcMI, DeadInsts); + return true; + } + } return tryFoldImplicitDef(MI, DeadInsts); } @@ -102,20 +141,22 @@ public: return false; Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c - unsigned TruncSrc; + Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy}}) || - isInstUnsupported({TargetOpcode::G_ASHR, {DstTy}}) || - isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + // Guess on the RHS shift amount type, which should be re-legalized if + // applicable. + if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy, DstTy}}) || + isInstUnsupported({TargetOpcode::G_ASHR, {DstTy, DstTy}}) || + isConstantUnsupported(DstTy)) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); LLT SrcTy = MRI.getType(SrcReg); - unsigned ShAmt = DstTy.getSizeInBits() - SrcTy.getSizeInBits(); + unsigned ShAmt = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits(); auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt); auto MIBShl = Builder.buildInstr( TargetOpcode::G_SHL, {DstTy}, @@ -138,7 +179,7 @@ public: if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(), MRI)) { Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); + Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); if (Opcode == TargetOpcode::G_ANYEXT) { @@ -150,7 +191,7 @@ public: } else { // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT. - if (isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + if (isConstantUnsupported(DstTy)) return false; LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;); Builder.buildConstant(DstReg, 0); @@ -162,6 +203,16 @@ public: return false; } + static unsigned getMergeOpcode(LLT OpTy, LLT DestTy) { + if (OpTy.isVector() && DestTy.isVector()) + return TargetOpcode::G_CONCAT_VECTORS; + + if (OpTy.isVector() && !DestTy.isVector()) + return TargetOpcode::G_BUILD_VECTOR; + + return TargetOpcode::G_MERGE_VALUES; + } + bool tryCombineMerges(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts) { @@ -169,27 +220,33 @@ public: return false; unsigned NumDefs = MI.getNumOperands() - 1; + MachineInstr *SrcDef = + getDefIgnoringCopies(MI.getOperand(NumDefs).getReg(), MRI); + if (!SrcDef) + return false; - unsigned MergingOpcode; LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg()); LLT DestTy = MRI.getType(MI.getOperand(0).getReg()); - if (OpTy.isVector() && DestTy.isVector()) - MergingOpcode = TargetOpcode::G_CONCAT_VECTORS; - else if (OpTy.isVector() && !DestTy.isVector()) - MergingOpcode = TargetOpcode::G_BUILD_VECTOR; - else - MergingOpcode = TargetOpcode::G_MERGE_VALUES; + MachineInstr *MergeI = SrcDef; + unsigned ConvertOp = 0; - MachineInstr *MergeI = - getOpcodeDef(MergingOpcode, MI.getOperand(NumDefs).getReg(), MRI); + // Handle intermediate conversions + unsigned SrcOp = SrcDef->getOpcode(); + if (isArtifactCast(SrcOp)) { + ConvertOp = SrcOp; + MergeI = getDefIgnoringCopies(SrcDef->getOperand(1).getReg(), MRI); + } - if (!MergeI) + // FIXME: Handle scalarizing concat_vectors (scalar result type with vector + // source) + unsigned MergingOpcode = getMergeOpcode(OpTy, DestTy); + if (!MergeI || MergeI->getOpcode() != MergingOpcode) return false; const unsigned NumMergeRegs = MergeI->getNumOperands() - 1; if (NumMergeRegs < NumDefs) { - if (NumDefs % NumMergeRegs != 0) + if (ConvertOp != 0 || NumDefs % NumMergeRegs != 0) return false; Builder.setInstr(MI); @@ -202,7 +259,7 @@ public: const unsigned NewNumDefs = NumDefs / NumMergeRegs; for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) { - SmallVector<unsigned, 2> DstRegs; + SmallVector<Register, 2> DstRegs; for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs; ++j, ++DefIdx) DstRegs.push_back(MI.getOperand(DefIdx).getReg()); @@ -211,7 +268,7 @@ public: } } else if (NumMergeRegs > NumDefs) { - if (NumMergeRegs % NumDefs != 0) + if (ConvertOp != 0 || NumMergeRegs % NumDefs != 0) return false; Builder.setInstr(MI); @@ -224,7 +281,7 @@ public: const unsigned NumRegs = NumMergeRegs / NumDefs; for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) { - SmallVector<unsigned, 2> Regs; + SmallVector<Register, 2> Regs; for (unsigned j = 0, Idx = NumRegs * DefIdx + 1; j < NumRegs; ++j, ++Idx) Regs.push_back(MergeI->getOperand(Idx).getReg()); @@ -233,10 +290,22 @@ public: } } else { + LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg()); + if (ConvertOp) { + Builder.setInstr(MI); + + for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { + Register MergeSrc = MergeI->getOperand(Idx + 1).getReg(); + Builder.buildInstr(ConvertOp, {MI.getOperand(Idx).getReg()}, + {MergeSrc}); + } + + markInstAndDefDead(MI, *MergeI, DeadInsts); + return true; + } // FIXME: is a COPY appropriate if the types mismatch? We know both // registers are allocatable by now. - if (MRI.getType(MI.getOperand(0).getReg()) != - MRI.getType(MergeI->getOperand(1).getReg())) + if (DestTy != MergeSrcTy) return false; for (unsigned Idx = 0; Idx < NumDefs; ++Idx) @@ -248,12 +317,77 @@ public: return true; } + static bool isMergeLikeOpcode(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_MERGE_VALUES: + case TargetOpcode::G_BUILD_VECTOR: + case TargetOpcode::G_CONCAT_VECTORS: + return true; + default: + return false; + } + } + + bool tryCombineExtract(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + assert(MI.getOpcode() == TargetOpcode::G_EXTRACT); + + // Try to use the source registers from a G_MERGE_VALUES + // + // %2 = G_MERGE_VALUES %0, %1 + // %3 = G_EXTRACT %2, N + // => + // + // for N < %2.getSizeInBits() / 2 + // %3 = G_EXTRACT %0, N + // + // for N >= %2.getSizeInBits() / 2 + // %3 = G_EXTRACT %1, (N - %0.getSizeInBits() + + unsigned Src = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + MachineInstr *MergeI = MRI.getVRegDef(Src); + if (!MergeI || !isMergeLikeOpcode(MergeI->getOpcode())) + return false; + + LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); + LLT SrcTy = MRI.getType(Src); + + // TODO: Do we need to check if the resulting extract is supported? + unsigned ExtractDstSize = DstTy.getSizeInBits(); + unsigned Offset = MI.getOperand(2).getImm(); + unsigned NumMergeSrcs = MergeI->getNumOperands() - 1; + unsigned MergeSrcSize = SrcTy.getSizeInBits() / NumMergeSrcs; + unsigned MergeSrcIdx = Offset / MergeSrcSize; + + // Compute the offset of the last bit the extract needs. + unsigned EndMergeSrcIdx = (Offset + ExtractDstSize - 1) / MergeSrcSize; + + // Can't handle the case where the extract spans multiple inputs. + if (MergeSrcIdx != EndMergeSrcIdx) + return false; + + // TODO: We could modify MI in place in most cases. + Builder.setInstr(MI); + Builder.buildExtract( + MI.getOperand(0).getReg(), + MergeI->getOperand(MergeSrcIdx + 1).getReg(), + Offset - MergeSrcIdx * MergeSrcSize); + markInstAndDefDead(MI, *MergeI, DeadInsts); + return true; + } + /// Try to combine away MI. /// Returns true if it combined away the MI. /// Adds instructions that are dead as a result of the combine /// into DeadInsts, which can include MI. bool tryCombineInstruction(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + GISelObserverWrapper &WrapperObserver) { + // This might be a recursive call, and we might have DeadInsts already + // populated. To avoid bad things happening later with multiple vreg defs + // etc, process the dead instructions now if any. + if (!DeadInsts.empty()) + deleteMarkedDeadInsts(DeadInsts, WrapperObserver); switch (MI.getOpcode()) { default: return false; @@ -265,16 +399,35 @@ public: return tryCombineSExt(MI, DeadInsts); case TargetOpcode::G_UNMERGE_VALUES: return tryCombineMerges(MI, DeadInsts); + case TargetOpcode::G_EXTRACT: + return tryCombineExtract(MI, DeadInsts); case TargetOpcode::G_TRUNC: { bool Changed = false; for (auto &Use : MRI.use_instructions(MI.getOperand(0).getReg())) - Changed |= tryCombineInstruction(Use, DeadInsts); + Changed |= tryCombineInstruction(Use, DeadInsts, WrapperObserver); return Changed; } } } private: + + static unsigned getArtifactSrcReg(const MachineInstr &MI) { + switch (MI.getOpcode()) { + case TargetOpcode::COPY: + case TargetOpcode::G_TRUNC: + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_ANYEXT: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_UNMERGE_VALUES: + return MI.getOperand(MI.getNumOperands() - 1).getReg(); + case TargetOpcode::G_EXTRACT: + return MI.getOperand(1).getReg(); + default: + llvm_unreachable("Not a legalization artifact happen"); + } + } + /// Mark MI as dead. If a def of one of MI's operands, DefMI, would also be /// dead due to MI being killed, then mark DefMI as dead too. /// Some of the combines (extends(trunc)), try to walk through redundant @@ -295,13 +448,15 @@ private: // and as a result, %3, %2, %1 are dead. MachineInstr *PrevMI = &MI; while (PrevMI != &DefMI) { - unsigned PrevRegSrc = - PrevMI->getOperand(PrevMI->getNumOperands() - 1).getReg(); + unsigned PrevRegSrc = getArtifactSrcReg(*PrevMI); + MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc); if (MRI.hasOneUse(PrevRegSrc)) { if (TmpDef != &DefMI) { - assert(TmpDef->getOpcode() == TargetOpcode::COPY && - "Expecting copy here"); + assert((TmpDef->getOpcode() == TargetOpcode::COPY || + isArtifactCast(TmpDef->getOpcode())) && + "Expecting copy or artifact cast here"); + DeadInsts.push_back(TmpDef); } } else @@ -312,6 +467,22 @@ private: DeadInsts.push_back(&DefMI); } + /// Erase the dead instructions in the list and call the observer hooks. + /// Normally the Legalizer will deal with erasing instructions that have been + /// marked dead. However, for the trunc(ext(x)) cases we can end up trying to + /// process instructions which have been marked dead, but otherwise break the + /// MIR by introducing multiple vreg defs. For those cases, allow the combines + /// to explicitly delete the instructions before we run into trouble. + void deleteMarkedDeadInsts(SmallVectorImpl<MachineInstr *> &DeadInsts, + GISelObserverWrapper &WrapperObserver) { + for (auto *DeadMI : DeadInsts) { + LLVM_DEBUG(dbgs() << *DeadMI << "Is dead, eagerly deleting\n"); + WrapperObserver.erasingInstr(*DeadMI); + DeadMI->eraseFromParentAndMarkDBGValuesForRemoval(); + } + DeadInsts.clear(); + } + /// Checks if the target legalizer info has specified anything about the /// instruction, or if unsupported. bool isInstUnsupported(const LegalityQuery &Query) const { @@ -320,10 +491,23 @@ private: return Step.Action == Unsupported || Step.Action == NotFound; } + bool isInstLegal(const LegalityQuery &Query) const { + return LI.getAction(Query).Action == LegalizeActions::Legal; + } + + bool isConstantUnsupported(LLT Ty) const { + if (!Ty.isVector()) + return isInstUnsupported({TargetOpcode::G_CONSTANT, {Ty}}); + + LLT EltTy = Ty.getElementType(); + return isInstUnsupported({TargetOpcode::G_CONSTANT, {EltTy}}) || + isInstUnsupported({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}); + } + /// Looks through copy instructions and returns the actual /// source register. - unsigned lookThroughCopyInstrs(unsigned Reg) { - unsigned TmpReg; + unsigned lookThroughCopyInstrs(Register Reg) { + Register TmpReg; while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) { if (MRI.getType(TmpReg).isValid()) Reg = TmpReg; diff --git a/include/llvm/CodeGen/GlobalISel/Legalizer.h b/include/llvm/CodeGen/GlobalISel/Legalizer.h index 8284ab6dac65e..13cf3f7e694d7 100644 --- a/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -1,9 +1,8 @@ -//== llvm/CodeGen/GlobalISel/LegalizePass.h ------------- -*- C++ -*-==// +//== llvm/CodeGen/GlobalISel/Legalizer.h ---------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -55,6 +54,11 @@ public: MachineFunctionProperties::Property::Legalized); } + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties() + .set(MachineFunctionProperties::Property::NoPHIs); + } + bool combineExtracts(MachineInstr &MI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII); diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 9b4ecf9284e31..a0f21e8b19d72 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -87,7 +86,7 @@ public: /// Legalize a vector instruction by increasing the number of vector elements /// involved and ignoring the added elements later. LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, - LLT WideTy); + LLT MoreTy); /// Expose MIRBuilder so clients can set their own RecordInsertInstruction /// functions @@ -105,19 +104,126 @@ private: unsigned ExtOpcode); /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and + /// replacing the vreg of the operand in place. + void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); + + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a /// Def by extending the operand's type to \p WideTy and truncating it back /// with the \p TruncOpcode, and replacing the vreg of the operand in place. void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, unsigned TruncOpcode = TargetOpcode::G_TRUNC); + // Legalize a single operand \p OpIdx of the machine instruction \p MI as a + // Def by truncating the operand's type to \p NarrowTy, replacing in place and + // extending back with \p ExtOpcode. + void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, + unsigned ExtOpcode); + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Def by performing it with additional vector elements and extracting the + /// result elements, and replacing the vreg of the operand in place. + void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); + + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Use by producing a vector with undefined high elements, extracting the + /// original vector type, and replacing the vreg of the operand in place. + void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); + + LegalizeResult + widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult + widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult + widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult + widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + /// Helper function to split a wide generic register into bitwise blocks with /// the given Type (which implies the number of blocks needed). The generic /// registers created are appended to Ops, starting at bit 0 of Reg. - void extractParts(unsigned Reg, LLT Ty, int NumParts, - SmallVectorImpl<unsigned> &VRegs); + void extractParts(Register Reg, LLT Ty, int NumParts, + SmallVectorImpl<Register> &VRegs); + + /// Version which handles irregular splits. + bool extractParts(Register Reg, LLT RegTy, LLT MainTy, + LLT &LeftoverTy, + SmallVectorImpl<Register> &VRegs, + SmallVectorImpl<Register> &LeftoverVRegs); + + /// Helper function to build a wide generic register \p DstReg of type \p + /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, + /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate + /// for the types. + /// + /// \p PartRegs must be registers of type \p PartTy. + /// + /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the + /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. + void insertParts(Register DstReg, LLT ResultTy, + LLT PartTy, ArrayRef<Register> PartRegs, + LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); + + /// Perform generic multiplication of values held in multiple registers. + /// Generated instructions use only types NarrowTy and i1. + /// Destination can be same or two times size of the source. + void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, + ArrayRef<Register> Src1Regs, + ArrayRef<Register> Src2Regs, LLT NarrowTy); + +public: + LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + /// Legalize a simple vector instruction where all operands are the same type + /// by splitting into multiple components. + LegalizeResult fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + + /// Legalize a instruction with a vector type where each operand may have a + /// different element type. All type indexes must have the same number of + /// elements. + LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + + LegalizeResult + fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult + fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult fewerElementsVectorPhi(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, + LLT MoreTy); + + LegalizeResult + reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, + LLT HalfTy, LLT ShiftAmtTy); + + LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); + LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + + LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); + LegalizeResult lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); + +private: MachineRegisterInfo &MRI; const LegalizerInfo &LI; /// To keep track of changes made by the LegalizerHelper. diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 13776dd3e87d1..513c98f2d23ff 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -93,6 +92,7 @@ enum LegalizeAction : std::uint8_t { UseLegacyRules, }; } // end namespace LegalizeActions +raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); using LegalizeActions::LegalizeAction; @@ -123,6 +123,7 @@ struct LegalityQuery { struct MemDesc { uint64_t SizeInBits; + uint64_t AlignInBits; AtomicOrdering Ordering; }; @@ -165,13 +166,23 @@ using LegalizeMutation = std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; namespace LegalityPredicates { -struct TypePairAndMemSize { +struct TypePairAndMemDesc { LLT Type0; LLT Type1; uint64_t MemSize; + uint64_t Align; + + bool operator==(const TypePairAndMemDesc &Other) const { + return Type0 == Other.Type0 && Type1 == Other.Type1 && + Align == Other.Align && + MemSize == Other.MemSize; + } - bool operator==(const TypePairAndMemSize &Other) const { + /// \returns true if this memory access is legal with for the acecss described + /// by \p Other (The alignment is sufficient for the size and result type). + bool isCompatible(const TypePairAndMemDesc &Other) const { return Type0 == Other.Type0 && Type1 == Other.Type1 && + Align >= Other.Align && MemSize == Other.MemSize; } }; @@ -200,20 +211,45 @@ typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, std::initializer_list<std::pair<LLT, LLT>> TypesInit); /// True iff the given types for the given pair of type indexes is one of the /// specified type pairs. -LegalityPredicate typePairAndMemSizeInSet( +LegalityPredicate typePairAndMemDescInSet( unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, - std::initializer_list<TypePairAndMemSize> TypesAndMemSizeInit); + std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit); /// True iff the specified type index is a scalar. LegalityPredicate isScalar(unsigned TypeIdx); +/// True iff the specified type index is a vector. +LegalityPredicate isVector(unsigned TypeIdx); +/// True iff the specified type index is a pointer (with any address space). +LegalityPredicate isPointer(unsigned TypeIdx); +/// True iff the specified type index is a pointer with the specified address +/// space. +LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); + /// True iff the specified type index is a scalar that's narrower than the given /// size. LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size); + /// True iff the specified type index is a scalar that's wider than the given /// size. LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size); + +/// True iff the specified type index is a scalar or vector with an element type +/// that's narrower than the given size. +LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); + +/// True iff the specified type index is a scalar or a vector with an element +/// type that's wider than the given size. +LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); + /// True iff the specified type index is a scalar whose size is not a power of /// 2. LegalityPredicate sizeNotPow2(unsigned TypeIdx); + +/// True iff the specified type index is a scalar or vector whose element size +/// is not a power of 2. +LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); + +/// True iff the specified type indices are both the same bit size. +LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); /// True iff the specified MMO index has a size that is not a power of 2 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); /// True iff the specified type index is a vector whose element count is not a @@ -228,13 +264,25 @@ LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, namespace LegalizeMutations { /// Select this specific type for the given type index. LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); + /// Keep the same type as the given type index. LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); -/// Widen the type for the given type index to the next power of 2. -LegalizeMutation widenScalarToNextPow2(unsigned TypeIdx, unsigned Min = 0); + +/// Keep the same scalar or element type as the given type index. +LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); + +/// Keep the same scalar or element type as the given type. +LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); + +/// Widen the scalar type or vector element type for the given type index to the +/// next power of 2. +LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); + /// Add more elements to the type for the given type index to the next power of /// 2. LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); +/// Break up the vector type for the given type index into the element type. +LegalizeMutation scalarize(unsigned TypeIdx); } // end namespace LegalizeMutations /// A single rule in a legalizer info ruleset. @@ -419,13 +467,13 @@ public: return actionFor(LegalizeAction::Legal, Types); } /// The instruction is legal when type indexes 0 and 1 along with the memory - /// size is any type and size tuple in the given list. - LegalizeRuleSet &legalForTypesWithMemSize( - std::initializer_list<LegalityPredicates::TypePairAndMemSize> - TypesAndMemSize) { + /// size and minimum alignment is any type and size tuple in the given list. + LegalizeRuleSet &legalForTypesWithMemDesc( + std::initializer_list<LegalityPredicates::TypePairAndMemDesc> + TypesAndMemDesc) { return actionIf(LegalizeAction::Legal, - LegalityPredicates::typePairAndMemSizeInSet( - typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemSize)); + LegalityPredicates::typePairAndMemDescInSet( + typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); } /// The instruction is legal when type indexes 0 and 1 are both in the given /// list. That is, the type pair is in the cartesian product of the list. @@ -438,6 +486,20 @@ public: std::initializer_list<LLT> Types1) { return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); } + /// The instruction is legal when type indexes 0, 1, and 2 are both their + /// respective lists. + LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1, + std::initializer_list<LLT> Types2) { + return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1, + Types2); + } + + LegalizeRuleSet &alwaysLegal() { + using namespace LegalizeMutations; + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Legal, always); + } /// The instruction is lowered. LegalizeRuleSet &lower() { @@ -588,6 +650,13 @@ public: LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { return actionFor(LegalizeAction::Custom, Types); } + + /// The instruction is custom when type indexes 0 and 1 is any type pair in the + /// given list. + LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) { + return actionFor(LegalizeAction::Custom, Types); + } + LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { return actionForCartesianProduct(LegalizeAction::Custom, Types); } @@ -597,13 +666,29 @@ public: return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); } + /// Unconditionally custom lower. + LegalizeRuleSet &custom() { + return customIf(always); + } + /// Widen the scalar to the next power of two that is at least MinSize. /// No effect if the type is not a scalar or is a power of two. LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize = 0) { using namespace LegalityPredicates; - return actionIf(LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), - LegalizeMutations::widenScalarToNextPow2(TypeIdx, MinSize)); + return actionIf( + LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); + } + + /// Widen the scalar or vector element type to the next power of two that is + /// at least MinSize. No effect if the scalar size is a power of two. + LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, + unsigned MinSize = 0) { + using namespace LegalityPredicates; + return actionIf( + LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); } LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { @@ -612,6 +697,32 @@ public: Mutation); } + LegalizeRuleSet &scalarize(unsigned TypeIdx) { + using namespace LegalityPredicates; + return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)), + LegalizeMutations::scalarize(TypeIdx)); + } + + /// Ensure the scalar or element is at least as wide as Ty. + LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::WidenScalar, + scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + + /// Ensure the scalar or element is at least as wide as Ty. + LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, + unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::WidenScalar, + all(Predicate, scalarOrEltNarrowerThan( + TypeIdx, Ty.getScalarSizeInBits())), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + /// Ensure the scalar is at least as wide as Ty. LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) { using namespace LegalityPredicates; @@ -622,6 +733,15 @@ public: } /// Ensure the scalar is at most as wide as Ty. + LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::NarrowScalar, + scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + + /// Ensure the scalar is at most as wide as Ty. LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT &Ty) { using namespace LegalityPredicates; using namespace LegalizeMutations; @@ -637,12 +757,12 @@ public: const LLT &Ty) { using namespace LegalityPredicates; using namespace LegalizeMutations; - return actionIf(LegalizeAction::NarrowScalar, - [=](const LegalityQuery &Query) { - return widerThan(TypeIdx, Ty.getSizeInBits()) && - Predicate(Query); - }, - changeTo(typeIdx(TypeIdx), Ty)); + return actionIf( + LegalizeAction::NarrowScalar, + [=](const LegalityQuery &Query) { + return widerThan(TypeIdx, Ty.getSizeInBits()) && Predicate(Query); + }, + changeElementTo(typeIdx(TypeIdx), Ty)); } /// Limit the range of scalar sizes to MinTy and MaxTy. @@ -652,6 +772,12 @@ public: return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); } + /// Limit the range of scalar sizes to MinTy and MaxTy. + LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT &MinTy, + const LLT &MaxTy) { + return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); + } + /// Widen the scalar to match the size of another. LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { typeIdx(TypeIdx); @@ -661,8 +787,25 @@ public: Query.Types[TypeIdx].getSizeInBits(); }, [=](const LegalityQuery &Query) { + LLT T = Query.Types[LargeTypeIdx]; return std::make_pair(TypeIdx, - Query.Types[LargeTypeIdx].getElementType()); + T.isVector() ? T.getElementType() : T); + }); + } + + /// Conditionally widen the scalar or elt to match the size of another. + LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, + unsigned TypeIdx, unsigned LargeTypeIdx) { + typeIdx(TypeIdx); + return widenScalarIf( + [=](const LegalityQuery &Query) { + return Query.Types[LargeTypeIdx].getScalarSizeInBits() > + Query.Types[TypeIdx].getScalarSizeInBits() && + Predicate(Query); + }, + [=](const LegalityQuery &Query) { + LLT T = Query.Types[LargeTypeIdx]; + return std::make_pair(TypeIdx, T); }); } @@ -691,7 +834,7 @@ public: [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; return std::make_pair( - TypeIdx, LLT::vector(MinElements, VecTy.getScalarSizeInBits())); + TypeIdx, LLT::vector(MinElements, VecTy.getElementType())); }); } /// Limit the number of elements in EltTy vectors to at most MaxElements. @@ -708,10 +851,8 @@ public: }, [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; - if (MaxElements == 1) - return std::make_pair(TypeIdx, VecTy.getElementType()); - return std::make_pair( - TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits())); + LLT NewTy = LLT::scalarOrVector(MaxElements, VecTy.getElementType()); + return std::make_pair(TypeIdx, NewTy); }); } /// Limit the number of elements for the given vectors to at least MinTy's @@ -962,12 +1103,22 @@ public: LegalizeActionStep getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; + bool isLegal(const LegalityQuery &Query) const { + return getAction(Query).Action == LegalizeAction::Legal; + } bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; + bool isLegalOrCustom(const MachineInstr &MI, + const MachineRegisterInfo &MRI) const; virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const; + /// Return true if MI is either legal or has been legalized and false + /// if not legal. + virtual bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const; + private: /// Determine what action should be taken to legalize the given generic /// instruction opcode, type-index and type. Requires computeTables to have diff --git a/include/llvm/CodeGen/GlobalISel/Localizer.h b/include/llvm/CodeGen/GlobalISel/Localizer.h index 1e2d4763e5e1d..06de5800b8b76 100644 --- a/include/llvm/CodeGen/GlobalISel/Localizer.h +++ b/include/llvm/CodeGen/GlobalISel/Localizer.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/GlobalISel/Localizer.h - Localizer -------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,12 +21,14 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H #define LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H +#include "llvm/ADT/SetVector.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { // Forward declarations. class MachineRegisterInfo; +class TargetTransformInfo; /// This pass implements the localization mechanism described at the /// top of this file. One specificity of the implementation is that @@ -44,9 +45,11 @@ private: /// MRI contains all the register class/bank information that this /// pass uses and updates. MachineRegisterInfo *MRI; + /// TTI used for getting remat costs for instructions. + TargetTransformInfo *TTI; /// Check whether or not \p MI needs to be moved close to its uses. - static bool shouldLocalize(const MachineInstr &MI); + bool shouldLocalize(const MachineInstr &MI); /// Check if \p MOUse is used in the same basic block as \p Def. /// If the use is in the same block, we say it is local. @@ -58,6 +61,15 @@ private: /// Initialize the field members using \p MF. void init(MachineFunction &MF); + typedef SmallSetVector<MachineInstr *, 32> LocalizedSetVecT; + + /// Do inter-block localization from the entry block. + bool localizeInterBlock(MachineFunction &MF, + LocalizedSetVecT &LocalizedInstrs); + + /// Do intra-block localization of already localized instructions. + bool localizeIntraBlock(LocalizedSetVecT &LocalizedInstrs); + public: Localizer(); diff --git a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index f77f9a8df7ee4..13eddd9539fa8 100644 --- a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -1,9 +1,8 @@ -//== ----- llvm/CodeGen/GlobalISel/MIPatternMatch.h --------------------- == // +//==------ llvm/CodeGen/GlobalISel/MIPatternMatch.h -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,8 +30,7 @@ template <typename SubPatternT> struct OneUse_match { SubPatternT SubPat; OneUse_match(const SubPatternT &SP) : SubPat(SP) {} - template <typename OpTy> - bool match(const MachineRegisterInfo &MRI, unsigned Reg) { + bool match(MachineRegisterInfo &MRI, unsigned Reg) { return MRI.hasOneUse(Reg) && SubPat.match(MRI, Reg); } }; @@ -162,7 +160,7 @@ template <typename Class> struct bind_ty { } }; -inline bind_ty<unsigned> m_Reg(unsigned &R) { return R; } +inline bind_ty<Register> m_Reg(Register &R) { return R; } inline bind_ty<MachineInstr *> m_MInstr(MachineInstr *&MI) { return MI; } inline bind_ty<LLT> m_Type(LLT &Ty) { return Ty; } diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 37de8f030410a..10d712176b1b3 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.h - MIBuilder --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -60,13 +59,15 @@ struct MachineIRBuilderState { class DstOp { union { LLT LLTTy; - unsigned Reg; + Register Reg; const TargetRegisterClass *RC; }; public: enum class DstType { Ty_LLT, Ty_Reg, Ty_RC }; DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {} + DstOp(Register R) : Reg(R), Ty(DstType::Ty_Reg) {} + DstOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(DstType::Ty_Reg) {} DstOp(const LLT &T) : LLTTy(T), Ty(DstType::Ty_LLT) {} DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {} @@ -96,7 +97,7 @@ public: llvm_unreachable("Unrecognised DstOp::DstType enum"); } - unsigned getReg() const { + Register getReg() const { assert(Ty == DstType::Ty_Reg && "Not a register"); return Reg; } @@ -119,13 +120,14 @@ private: class SrcOp { union { MachineInstrBuilder SrcMIB; - unsigned Reg; + Register Reg; CmpInst::Predicate Pred; }; public: enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate }; - SrcOp(unsigned R) : Reg(R), Ty(SrcType::Ty_Reg) {} + SrcOp(Register R) : Reg(R), Ty(SrcType::Ty_Reg) {} + SrcOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(SrcType::Ty_Reg) {} SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {} SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {} @@ -155,7 +157,7 @@ public: llvm_unreachable("Unrecognised SrcOp::SrcType enum"); } - unsigned getReg() const { + Register getReg() const { switch (Ty) { case SrcType::Ty_Predicate: llvm_unreachable("Not a register operand"); @@ -202,6 +204,7 @@ protected: void validateTruncExt(const LLT &Dst, const LLT &Src, bool IsExtend); void validateBinaryOp(const LLT &Res, const LLT &Op0, const LLT &Op1); + void validateShiftOp(const LLT &Res, const LLT &Op0, const LLT &Op1); void validateSelectOp(const LLT &ResTy, const LLT &TstTy, const LLT &Op0Ty, const LLT &Op1Ty); @@ -230,6 +233,15 @@ public: return *State.MF; } + const MachineFunction &getMF() const { + assert(State.MF && "MachineFunction is not set"); + return *State.MF; + } + + const DataLayout &getDataLayout() const { + return getMF().getFunction().getParent()->getDataLayout(); + } + /// Getter for DebugLoc const DebugLoc &getDL() { return State.DL; } @@ -310,13 +322,13 @@ public: /// Build and insert a DBG_VALUE instruction expressing the fact that the /// associated \p Variable lives in \p Reg (suitably modified by \p Expr). - MachineInstrBuilder buildDirectDbgValue(unsigned Reg, const MDNode *Variable, + MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr); /// Build and insert a DBG_VALUE instruction expressing the fact that the /// associated \p Variable lives in memory at \p Reg (suitably modified by \p /// Expr). - MachineInstrBuilder buildIndirectDbgValue(unsigned Reg, + MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr); @@ -345,7 +357,7 @@ public: /// \pre \p Res must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildFrameIndex(unsigned Res, int Idx); + MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx); /// Build and insert \p Res = G_GLOBAL_VALUE \p GV /// @@ -357,8 +369,7 @@ public: /// in the same address space as \p GV. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildGlobalValue(unsigned Res, const GlobalValue *GV); - + MachineInstrBuilder buildGlobalValue(const DstOp &Res, const GlobalValue *GV); /// Build and insert \p Res = G_GEP \p Op0, \p Op1 /// @@ -371,8 +382,8 @@ public: /// \pre \p Op1 must be a generic virtual register with scalar type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildGEP(unsigned Res, unsigned Op0, - unsigned Op1); + MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0, + const SrcOp &Op1); /// Materialize and insert \p Res = G_GEP \p Op0, (G_CONSTANT \p Value) /// @@ -390,7 +401,7 @@ public: /// type as \p Op0 or \p Op0 itself. /// /// \return a MachineInstrBuilder for the newly created instruction. - Optional<MachineInstrBuilder> materializeGEP(unsigned &Res, unsigned Op0, + Optional<MachineInstrBuilder> materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value); @@ -407,9 +418,24 @@ public: /// be cleared in \p Op0. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildPtrMask(unsigned Res, unsigned Op0, + MachineInstrBuilder buildPtrMask(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits); + /// Build and insert \p Res, \p CarryOut = G_UADDO \p Op0, \p Op1 + /// + /// G_UADDO sets \p Res to \p Op0 + \p Op1 (truncated to the bit width) and + /// sets \p CarryOut to 1 if the result overflowed in unsigned arithmetic. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers with the + /// same scalar type. + ////\pre \p CarryOut must be generic virtual register with scalar type + ///(typically s1) + /// + /// \return The newly created instruction. + MachineInstrBuilder buildUAddo(const DstOp &Res, const DstOp &CarryOut, + const SrcOp &Op0, const SrcOp &Op1); + /// Build and insert \p Res, \p CarryOut = G_UADDE \p Op0, /// \p Op1, \p CarryIn /// @@ -458,6 +484,25 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op); + /// Build and insert a G_PTRTOINT instruction. + MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_PTRTOINT, {Dst}, {Src}); + } + + /// Build and insert \p Dst = G_BITCAST \p Src + MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_BITCAST, {Dst}, {Src}); + } + + /// \return The opcode of the extension the target wants to use for boolean + /// values. + unsigned getBoolExtOp(bool IsVec, bool IsFP) const; + + // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_SEXT \p Op, or \p Res + // = G_ZEXT \p Op depending on how the target wants to extend boolean values. + MachineInstrBuilder buildBoolExt(const DstOp &Res, const SrcOp &Op, + bool IsFP); + /// Build and insert \p Res = G_ZEXT \p Op /// /// G_ZEXT produces a register of the specified width, with bits 0 to @@ -538,7 +583,7 @@ public: /// depend on bit 0 (for now). /// /// \return The newly created instruction. - MachineInstrBuilder buildBrCond(unsigned Tst, MachineBasicBlock &Dest); + MachineInstrBuilder buildBrCond(Register Tst, MachineBasicBlock &Dest); /// Build and insert G_BRINDIRECT \p Tgt /// @@ -548,7 +593,21 @@ public: /// \pre \p Tgt must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildBrIndirect(unsigned Tgt); + MachineInstrBuilder buildBrIndirect(Register Tgt); + + /// Build and insert G_BRJT \p TablePtr, \p JTI, \p IndexReg + /// + /// G_BRJT is a jump table branch using a table base pointer \p TablePtr, + /// jump table index \p JTI and index \p IndexReg + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p TablePtr must be a generic virtual register with pointer type. + /// \pre \p JTI must be be a jump table index. + /// \pre \p IndexReg must be a generic virtual register with pointer type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, + Register IndexReg); /// Build and insert \p Res = G_CONSTANT \p Val /// @@ -572,6 +631,7 @@ public: /// /// \return The newly created instruction. MachineInstrBuilder buildConstant(const DstOp &Res, int64_t Val); + MachineInstrBuilder buildConstant(const DstOp &Res, const APInt &Val); /// Build and insert \p Res = G_FCONSTANT \p Val /// @@ -586,6 +646,7 @@ public: const ConstantFP &Val); MachineInstrBuilder buildFConstant(const DstOp &Res, double Val); + MachineInstrBuilder buildFConstant(const DstOp &Res, const APFloat &Val); /// Build and insert \p Res = COPY Op /// @@ -605,7 +666,7 @@ public: /// \pre \p Addr must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildLoad(unsigned Res, unsigned Addr, + MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO); /// Build and insert `Res = <opcode> Addr, MMO`. @@ -617,8 +678,8 @@ public: /// \pre \p Addr must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildLoadInstr(unsigned Opcode, unsigned Res, - unsigned Addr, MachineMemOperand &MMO); + MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, + const SrcOp &Addr, MachineMemOperand &MMO); /// Build and insert `G_STORE Val, Addr, MMO`. /// @@ -629,7 +690,7 @@ public: /// \pre \p Addr must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildStore(unsigned Val, unsigned Addr, + MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO); /// Build and insert `Res0, ... = G_EXTRACT Src, Idx0`. @@ -638,7 +699,7 @@ public: /// \pre \p Res and \p Src must be generic virtual registers. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildExtract(unsigned Res, unsigned Src, uint64_t Index); + MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index); /// Build and insert \p Res = IMPLICIT_DEF. MachineInstrBuilder buildUndef(const DstOp &Res); @@ -656,7 +717,7 @@ public: /// \pre The bits defined by each Op (derived from index and scalar size) must /// not overlap. /// \pre \p Indices must be in ascending order of bit position. - void buildSequence(unsigned Res, ArrayRef<unsigned> Ops, + void buildSequence(Register Res, ArrayRef<Register> Ops, ArrayRef<uint64_t> Indices); /// Build and insert \p Res = G_MERGE_VALUES \p Op0, ... @@ -670,7 +731,7 @@ public: /// \pre The type of all \p Ops registers must be identical. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef<unsigned> Ops); + MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef<Register> Ops); /// Build and insert \p Res0, ... = G_UNMERGE_VALUES \p Op /// @@ -683,7 +744,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildUnmerge(ArrayRef<LLT> Res, const SrcOp &Op); - MachineInstrBuilder buildUnmerge(ArrayRef<unsigned> Res, const SrcOp &Op); + MachineInstrBuilder buildUnmerge(ArrayRef<Register> Res, const SrcOp &Op); + + /// Build and insert an unmerge of \p Res sized pieces to cover \p Op + MachineInstrBuilder buildUnmerge(LLT Res, const SrcOp &Op); /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... /// @@ -695,7 +759,12 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildBuildVector(const DstOp &Res, - ArrayRef<unsigned> Ops); + ArrayRef<Register> Ops); + + /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill + /// the number of elements + MachineInstrBuilder buildSplatVector(const DstOp &Res, + const SrcOp &Src); /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... /// @@ -711,7 +780,7 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildBuildVectorTrunc(const DstOp &Res, - ArrayRef<unsigned> Ops); + ArrayRef<Register> Ops); /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... /// @@ -725,10 +794,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildConcatVectors(const DstOp &Res, - ArrayRef<unsigned> Ops); + ArrayRef<Register> Ops); - MachineInstrBuilder buildInsert(unsigned Res, unsigned Src, - unsigned Op, unsigned Index); + MachineInstrBuilder buildInsert(Register Res, Register Src, + Register Op, unsigned Index); /// Build and insert either a G_INTRINSIC (if \p HasSideEffects is false) or /// G_INTRINSIC_W_SIDE_EFFECTS instruction. Its first operand will be the @@ -740,7 +809,9 @@ public: /// \pre setBasicBlock or setMI must have been called. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, unsigned Res, + MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<Register> Res, + bool HasSideEffects); + MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<DstOp> Res, bool HasSideEffects); /// Build and insert \p Res = G_FPTRUNC \p Op @@ -855,8 +926,8 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder - buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes, - unsigned Addr, unsigned CmpVal, unsigned NewVal, + buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, + Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, @@ -873,8 +944,8 @@ public: /// registers of the same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, - unsigned CmpVal, unsigned NewVal, + MachineInstrBuilder buildAtomicCmpXchg(Register OldValRes, Register Addr, + Register CmpVal, Register NewVal, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO`. @@ -890,8 +961,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes, - unsigned Addr, unsigned Val, + MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes, + Register Addr, Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`. @@ -906,8 +977,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWXchg(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_ADD Addr, Val, MMO`. /// @@ -921,8 +992,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWAdd(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_SUB Addr, Val, MMO`. /// @@ -936,8 +1007,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWSub(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_AND Addr, Val, MMO`. /// @@ -951,8 +1022,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWAnd(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_NAND Addr, Val, MMO`. /// @@ -967,8 +1038,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWNand(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_OR Addr, Val, MMO`. /// @@ -982,8 +1053,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWOr(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_XOR Addr, Val, MMO`. /// @@ -997,8 +1068,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWXor(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_MAX Addr, Val, MMO`. /// @@ -1013,8 +1084,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWMax(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_MIN Addr, Val, MMO`. /// @@ -1029,8 +1100,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWMin(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMAX Addr, Val, MMO`. /// @@ -1045,8 +1116,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWUmax(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMIN Addr, Val, MMO`. /// @@ -1061,8 +1132,11 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWUmin(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); + + /// Build and insert `G_FENCE Ordering, Scope`. + MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope); /// Build and insert \p Res = G_BLOCK_ADDR \p BA /// @@ -1072,7 +1146,7 @@ public: /// \pre \p Res must be a generic virtual register of a pointer type. /// /// \return The newly created instruction. - MachineInstrBuilder buildBlockAddress(unsigned Res, const BlockAddress *BA); + MachineInstrBuilder buildBlockAddress(Register Res, const BlockAddress *BA); /// Build and insert \p Res = G_ADD \p Op0, \p Op1 /// @@ -1124,6 +1198,36 @@ public: return buildInstr(TargetOpcode::G_MUL, {Dst}, {Src0, Src1}, Flags); } + MachineInstrBuilder buildUMulH(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_UMULH, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildSMulH(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_SHL, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_LSHR, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_ASHR, {Dst}, {Src0, Src1}, Flags); + } + /// Build and insert \p Res = G_AND \p Op0, \p Op1 /// /// G_AND sets \p Res to the bitwise and of integer parameters \p Op0 and \p @@ -1155,6 +1259,137 @@ public: return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1}); } + /// Build and insert \p Res = G_XOR \p Op0, \p Op1 + MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_XOR, {Dst}, {Src0, Src1}); + } + + /// Build and insert a bitwise not, + /// \p NegOne = G_CONSTANT -1 + /// \p Res = G_OR \p Op0, NegOne + MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0) { + auto NegOne = buildConstant(Dst.getLLTTy(*getMRI()), -1); + return buildInstr(TargetOpcode::G_XOR, {Dst}, {Src0, NegOne}); + } + + /// Build and insert \p Res = G_CTPOP \p Op0, \p Src0 + MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTPOP, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTLZ \p Op0, \p Src0 + MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTLZ, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTLZ_ZERO_UNDEF \p Op0, \p Src0 + MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTTZ \p Op0, \p Src0 + MachineInstrBuilder buildCTTZ(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTTZ, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTTZ_ZERO_UNDEF \p Op0, \p Src0 + MachineInstrBuilder buildCTTZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FADD \p Op0, \p Op1 + MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_FSUB \p Op0, \p Op1 + MachineInstrBuilder buildFSub(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_FSUB, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_FMA \p Op0, \p Op1, \p Op2 + MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, const SrcOp &Src2) { + return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2}); + } + + /// Build and insert \p Res = G_FNEG \p Op0 + MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FABS \p Op0 + MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0}); + } + + /// Build and insert \p Dst = G_FCANONICALIZE \p Src0 + MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FCANONICALIZE, {Dst}, {Src0}, Flags); + } + + /// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1 + MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_FCOPYSIGN, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_UITOFP \p Src0 + MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_UITOFP, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_SITOFP \p Src0 + MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_SITOFP, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FPTOUI \p Src0 + MachineInstrBuilder buildFPTOUI(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FPTOUI, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FPTOSI \p Src0 + MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FPTOSI, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_SMIN \p Op0, \p Op1 + MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_SMIN, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_SMAX \p Op0, \p Op1 + MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_SMAX, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_UMIN \p Op0, \p Op1 + MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_UMIN, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_UMAX \p Op0, \p Op1 + MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_UMAX, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_JUMP_TABLE \p JTI + /// + /// G_JUMP_TABLE sets \p Res to the address of the jump table specified by + /// the jump table index \p JTI. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI); + virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps, ArrayRef<SrcOp> SrcOps, Optional<unsigned> Flags = None); diff --git a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index c53ae416e60b3..d9d076ba312c0 100644 --- a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -1,9 +1,8 @@ //=- llvm/CodeGen/GlobalISel/RegBankSelect.h - Reg Bank Selector --*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -524,7 +523,7 @@ private: /// \p OnlyAssign == true means that \p Reg just needs to be assigned a /// register bank. I.e., no repairing is necessary to have the /// assignment match. - bool assignmentMatch(unsigned Reg, + bool assignmentMatch(Register Reg, const RegisterBankInfo::ValueMapping &ValMapping, bool &OnlyAssign) const; @@ -563,7 +562,7 @@ private: bool repairReg(MachineOperand &MO, const RegisterBankInfo::ValueMapping &ValMapping, RegBankSelect::RepairingPlacement &RepairPt, - const iterator_range<SmallVectorImpl<unsigned>::const_iterator> + const iterator_range<SmallVectorImpl<Register>::const_iterator> &NewVRegs); /// Return the cost of the instruction needed to map \p MO to \p ValMapping. @@ -634,6 +633,11 @@ public: MachineFunctionProperties::Property::RegBankSelected); } + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties() + .set(MachineFunctionProperties::Property::NoPHIs); + } + /// Walk through \p MF and assign a register bank to every virtual register /// that are still mapped to nothing. /// The target needs to provide a RegisterBankInfo and in particular diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBank.h b/include/llvm/CodeGen/GlobalISel/RegisterBank.h index d5612e17393cc..f528d1a460128 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBank.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBank.h @@ -1,9 +1,8 @@ //==-- llvm/CodeGen/GlobalISel/RegisterBank.h - Register Bank ----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index c33b32b2db401..e84b1c3ea8b15 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/RegisterBankInfo.h ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/CodeGen/Register.h" #include "llvm/Support/ErrorHandling.h" #include <cassert> #include <initializer_list> @@ -161,6 +161,10 @@ public: const PartialMapping *begin() const { return BreakDown; } const PartialMapping *end() const { return BreakDown + NumBreakDowns; } + /// \return true if all partial mappings are the same size and register + /// bank. + bool partsAllUniform() const; + /// Check if this ValueMapping is valid. bool isValid() const { return BreakDown && NumBreakDowns; } @@ -190,7 +194,7 @@ public: unsigned Cost = 0; /// Mapping of all the operands. - const ValueMapping *OperandsMapping; + const ValueMapping *OperandsMapping = nullptr; /// Number of operands. unsigned NumOperands = 0; @@ -207,15 +211,11 @@ public: /// The rationale is that it is more efficient for the optimizers /// to be able to assume that the mapping of the ith operand is /// at the index i. - /// - /// \pre ID != InvalidMappingID InstructionMapping(unsigned ID, unsigned Cost, const ValueMapping *OperandsMapping, unsigned NumOperands) : ID(ID), Cost(Cost), OperandsMapping(OperandsMapping), NumOperands(NumOperands) { - assert(getID() != InvalidMappingID && - "Use the default constructor for invalid mapping"); } /// Default constructor. @@ -282,7 +282,7 @@ public: SmallVector<int, 8> OpToNewVRegIdx; /// Hold the registers that will be used to map MI with InstrMapping. - SmallVector<unsigned, 8> NewVRegs; + SmallVector<Register, 8> NewVRegs; /// Current MachineRegisterInfo, used to create new virtual registers. MachineRegisterInfo &MRI; @@ -303,15 +303,15 @@ public: /// \return The iterator range for the space created. // /// \pre getMI().getOperand(OpIdx).isReg() - iterator_range<SmallVectorImpl<unsigned>::iterator> + iterator_range<SmallVectorImpl<Register>::iterator> getVRegsMem(unsigned OpIdx); /// Get the end iterator for a range starting at \p StartIdx and /// spannig \p NumVal in NewVRegs. /// \pre StartIdx + NumVal <= NewVRegs.size() - SmallVectorImpl<unsigned>::const_iterator + SmallVectorImpl<Register>::const_iterator getNewVRegsEnd(unsigned StartIdx, unsigned NumVal) const; - SmallVectorImpl<unsigned>::iterator getNewVRegsEnd(unsigned StartIdx, + SmallVectorImpl<Register>::iterator getNewVRegsEnd(unsigned StartIdx, unsigned NumVal); public: @@ -357,7 +357,7 @@ public: /// /// \post the \p PartialMapIdx-th register of the value mapping of the \p /// OpIdx-th operand has been set. - void setVRegs(unsigned OpIdx, unsigned PartialMapIdx, unsigned NewVReg); + void setVRegs(unsigned OpIdx, unsigned PartialMapIdx, Register NewVReg); /// Get all the virtual registers required to map the \p OpIdx-th operand of /// the instruction. @@ -371,7 +371,7 @@ public: /// /// \pre getMI().getOperand(OpIdx).isReg() /// \pre ForDebug || All partial mappings have been set a register - iterator_range<SmallVectorImpl<unsigned>::const_iterator> + iterator_range<SmallVectorImpl<Register>::const_iterator> getVRegs(unsigned OpIdx, bool ForDebug = false) const; /// Print this operands mapper on dbgs() stream. @@ -435,7 +435,7 @@ protected: /// Get the MinimalPhysRegClass for Reg. /// \pre Reg is a physical register. const TargetRegisterClass & - getMinimalPhysRegClass(unsigned Reg, const TargetRegisterInfo &TRI) const; + getMinimalPhysRegClass(Register Reg, const TargetRegisterInfo &TRI) const; /// Try to get the mapping of \p MI. /// See getInstrMapping for more details on what a mapping represents. @@ -580,7 +580,7 @@ public: /// or a register bank, then this returns nullptr. /// /// \pre Reg != 0 (NoRegister) - const RegisterBank *getRegBank(unsigned Reg, const MachineRegisterInfo &MRI, + const RegisterBank *getRegBank(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const; /// Get the total number of register banks. @@ -618,6 +618,21 @@ public: return &A != &B; } + /// \returns true if emitting a copy from \p Src to \p Dst is impossible. + bool cannotCopy(const RegisterBank &Dst, const RegisterBank &Src, + unsigned Size) const { + return copyCost(Dst, Src, Size) == std::numeric_limits<unsigned>::max(); + } + + /// Get the cost of using \p ValMapping to decompose a register. This is + /// similar to ::copyCost, except for cases where multiple copy-like + /// operations need to be inserted. If the register is used as a source + /// operand and already has a bank assigned, \p CurBank is non-null. + virtual unsigned getBreakDownCost(const ValueMapping &ValMapping, + const RegisterBank *CurBank = nullptr) const { + return std::numeric_limits<unsigned>::max(); + } + /// Constrain the (possibly generic) virtual register \p Reg to \p RC. /// /// \pre \p Reg is a virtual register that either has a bank or a class. @@ -626,7 +641,7 @@ public: /// \note Use MachineRegisterInfo::constrainRegAttrs instead for any non-isel /// purpose, including non-select passes of GlobalISel static const TargetRegisterClass * - constrainGenericRegister(unsigned Reg, const TargetRegisterClass &RC, + constrainGenericRegister(Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI); /// Identifier used when the related instruction mapping instance @@ -711,7 +726,7 @@ public: /// virtual register. /// /// \pre \p Reg != 0 (NoRegister). - unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, + unsigned getSizeInBits(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const; /// Check that information hold by this instance make sense for the diff --git a/include/llvm/CodeGen/GlobalISel/Types.h b/include/llvm/CodeGen/GlobalISel/Types.h index 7b22e343a7f86..4fd7043ba02de 100644 --- a/include/llvm/CodeGen/GlobalISel/Types.h +++ b/include/llvm/CodeGen/GlobalISel/Types.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/Types.h - Types used by GISel ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h index 82b791d35b2b5..4cdaa48fb6899 100644 --- a/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/include/llvm/CodeGen/GlobalISel/Utils.h @@ -1,9 +1,8 @@ //==-- llvm/CodeGen/GlobalISel/Utils.h ---------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #define LLVM_CODEGEN_GLOBALISEL_UTILS_H #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/Register.h" namespace llvm { @@ -37,21 +37,37 @@ class ConstantFP; class APFloat; /// Try to constrain Reg to the specified register class. If this fails, -/// create a new virtual register in the correct class and insert a COPY before -/// \p InsertPt. The debug location of \p InsertPt is used for the new copy. +/// create a new virtual register in the correct class. /// /// \return The virtual register constrained to the right register class. unsigned constrainRegToClass(MachineRegisterInfo &MRI, const TargetInstrInfo &TII, - const RegisterBankInfo &RBI, - MachineInstr &InsertPt, unsigned Reg, + const RegisterBankInfo &RBI, unsigned Reg, const TargetRegisterClass &RegClass); +/// Constrain the Register operand OpIdx, so that it is now constrained to the +/// TargetRegisterClass passed as an argument (RegClass). +/// If this fails, create a new virtual register in the correct class and +/// insert a COPY before \p InsertPt if it is a use or after if it is a +/// definition. The debug location of \p InsertPt is used for the new copy. +/// +/// \return The virtual register constrained to the right register class. +unsigned constrainOperandRegClass(const MachineFunction &MF, + const TargetRegisterInfo &TRI, + MachineRegisterInfo &MRI, + const TargetInstrInfo &TII, + const RegisterBankInfo &RBI, + MachineInstr &InsertPt, + const TargetRegisterClass &RegClass, + const MachineOperand &RegMO, unsigned OpIdx); + /// Try to constrain Reg so that it is usable by argument OpIdx of the /// provided MCInstrDesc \p II. If this fails, create a new virtual -/// register in the correct class and insert a COPY before \p InsertPt. -/// This is equivalent to constrainRegToClass() with RegClass obtained from the -/// MCInstrDesc. The debug location of \p InsertPt is used for the new copy. +/// register in the correct class and insert a COPY before \p InsertPt +/// if it is a use or after if it is a definition. +/// This is equivalent to constrainOperandRegClass(..., RegClass, ...) +/// with RegClass obtained from the MCInstrDesc. The debug location of \p +/// InsertPt is used for the new copy. /// /// \return The virtual register constrained to the right register class. unsigned constrainOperandRegClass(const MachineFunction &MF, @@ -90,17 +106,40 @@ void reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, const char *PassName, StringRef Msg, const MachineInstr &MI); +/// If \p VReg is defined by a G_CONSTANT fits in int64_t +/// returns it. Optional<int64_t> getConstantVRegVal(unsigned VReg, const MachineRegisterInfo &MRI); +/// Simple struct used to hold a constant integer value and a virtual +/// register. +struct ValueAndVReg { + int64_t Value; + unsigned VReg; +}; +/// If \p VReg is defined by a statically evaluable chain of +/// instructions rooted on a G_CONSTANT (\p LookThroughInstrs == true) +/// and that constant fits in int64_t, returns its value as well as +/// the virtual register defined by this G_CONSTANT. +/// When \p LookThroughInstrs == false, this function behaves like +/// getConstantVRegVal. +Optional<ValueAndVReg> +getConstantVRegValWithLookThrough(unsigned VReg, const MachineRegisterInfo &MRI, + bool LookThroughInstrs = true); const ConstantFP* getConstantFPVRegVal(unsigned VReg, const MachineRegisterInfo &MRI); /// See if Reg is defined by an single def instruction that is /// Opcode. Also try to do trivial folding if it's a COPY with /// same types. Returns null otherwise. -MachineInstr *getOpcodeDef(unsigned Opcode, unsigned Reg, +MachineInstr *getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI); +/// Find the def instruction for \p Reg, folding away any trivial copies. Note +/// it may still return a COPY, if it changes the type. May return nullptr if \p +/// Reg is not a generic virtual register. +MachineInstr *getDefIgnoringCopies(Register Reg, + const MachineRegisterInfo &MRI); + /// Returns an APFloat from Val converted to the appropriate size. APFloat getAPFloatFromSize(double Val, unsigned Size); @@ -111,5 +150,16 @@ void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, const unsigned Op2, const MachineRegisterInfo &MRI); + +/// Returns true if \p Val can be assumed to never be a NaN. If \p SNaN is true, +/// this returns if \p Val can be assumed to never be a signaling NaN. +bool isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, + bool SNaN = false); + +/// Returns true if \p Val can be assumed to never be a signaling NaN. +inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) { + return isKnownNeverNaN(Val, MRI, true); +} + } // End namespace llvm. #endif |