aboutsummaryrefslogtreecommitdiff
path: root/include/llvm/CodeGen/GlobalISel
diff options
context:
space:
mode:
Diffstat (limited to 'include/llvm/CodeGen/GlobalISel')
-rw-r--r--include/llvm/CodeGen/GlobalISel/CallLowering.h127
-rw-r--r--include/llvm/CodeGen/GlobalISel/CombinerHelper.h127
-rw-r--r--include/llvm/CodeGen/GlobalISel/CombinerInfo.h15
-rw-r--r--include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h11
-rw-r--r--include/llvm/CodeGen/GlobalISel/GISelKnownBits.h111
-rw-r--r--include/llvm/CodeGen/GlobalISel/IRTranslator.h12
-rw-r--r--include/llvm/CodeGen/GlobalISel/InstructionSelector.h34
-rw-r--r--include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h66
-rw-r--r--include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h92
-rw-r--r--include/llvm/CodeGen/GlobalISel/LegalizerHelper.h20
-rw-r--r--include/llvm/CodeGen/GlobalISel/LegalizerInfo.h61
-rw-r--r--include/llvm/CodeGen/GlobalISel/MIPatternMatch.h20
-rw-r--r--include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h93
-rw-r--r--include/llvm/CodeGen/GlobalISel/Utils.h22
14 files changed, 682 insertions, 129 deletions
diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h
index d717121ad78e..4901a3748e4a 100644
--- a/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -45,18 +45,62 @@ class CallLowering {
public:
struct ArgInfo {
SmallVector<Register, 4> Regs;
+ // If the argument had to be split into multiple parts according to the
+ // target calling convention, then this contains the original vregs
+ // if the argument was an incoming arg.
+ SmallVector<Register, 2> OrigRegs;
Type *Ty;
- ISD::ArgFlagsTy Flags;
+ SmallVector<ISD::ArgFlagsTy, 4> Flags;
bool IsFixed;
ArgInfo(ArrayRef<Register> Regs, Type *Ty,
- ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, bool IsFixed = true)
- : Regs(Regs.begin(), Regs.end()), Ty(Ty), Flags(Flags),
- IsFixed(IsFixed) {
+ ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(),
+ bool IsFixed = true)
+ : Regs(Regs.begin(), Regs.end()), Ty(Ty),
+ Flags(Flags.begin(), Flags.end()), IsFixed(IsFixed) {
+ if (!Regs.empty() && Flags.empty())
+ this->Flags.push_back(ISD::ArgFlagsTy());
// FIXME: We should have just one way of saying "no register".
assert((Ty->isVoidTy() == (Regs.empty() || Regs[0] == 0)) &&
"only void types should have no register");
}
+
+ ArgInfo() : Ty(nullptr), IsFixed(false) {}
+ };
+
+ struct CallLoweringInfo {
+ /// Calling convention to be used for the call.
+ CallingConv::ID CallConv = CallingConv::C;
+
+ /// Destination of the call. It should be either a register, globaladdress,
+ /// or externalsymbol.
+ MachineOperand Callee = MachineOperand::CreateImm(0);
+
+ /// Descriptor for the return type of the function.
+ ArgInfo OrigRet;
+
+ /// List of descriptors of the arguments passed to the function.
+ SmallVector<ArgInfo, 8> OrigArgs;
+
+ /// Valid if the call has a swifterror inout parameter, and contains the
+ /// vreg that the swifterror should be copied into after the call.
+ Register SwiftErrorVReg = 0;
+
+ MDNode *KnownCallees = nullptr;
+
+ /// True if the call must be tail call optimized.
+ bool IsMustTailCall = false;
+
+ /// True if the call passes all target-independent checks for tail call
+ /// optimization.
+ bool IsTailCall = false;
+
+ /// True if the call was lowered as a tail call. This is consumed by the
+ /// legalizer. This allows the legalizer to lower libcalls as tail calls.
+ bool LoweredTailCall = false;
+
+ /// True if the call is to a vararg function.
+ bool IsVarArg = false;
};
/// Argument handling is mostly uniform between the four places that
@@ -72,9 +116,9 @@ public:
virtual ~ValueHandler() = default;
- /// Returns true if the handler is dealing with formal arguments,
- /// not with return values etc.
- virtual bool isArgumentHandler() const { return false; }
+ /// Returns true if the handler is dealing with incoming arguments,
+ /// i.e. those that move values from some physical location to vregs.
+ virtual bool isIncomingArgumentHandler() const = 0;
/// Materialize a VReg containing the address of the specified
/// stack-based object. This is either based on a FrameIndex or
@@ -112,8 +156,8 @@ public:
virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
CCValAssign::LocInfo LocInfo, const ArgInfo &Info,
- CCState &State) {
- return AssignFn(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State);
+ ISD::ArgFlagsTy Flags, CCState &State) {
+ return AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
}
MachineIRBuilder &MIRBuilder;
@@ -162,12 +206,42 @@ protected:
/// \p Callback to move them to the assigned locations.
///
/// \return True if everything has succeeded, false otherwise.
- bool handleAssignments(MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args,
+ bool handleAssignments(MachineIRBuilder &MIRBuilder,
+ SmallVectorImpl<ArgInfo> &Args,
ValueHandler &Handler) const;
bool handleAssignments(CCState &CCState,
SmallVectorImpl<CCValAssign> &ArgLocs,
- MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args,
+ MachineIRBuilder &MIRBuilder,
+ SmallVectorImpl<ArgInfo> &Args,
ValueHandler &Handler) const;
+
+ /// Analyze passed or returned values from a call, supplied in \p ArgInfo,
+ /// incorporating info about the passed values into \p CCState.
+ ///
+ /// Used to check if arguments are suitable for tail call lowering.
+ bool analyzeArgInfo(CCState &CCState, SmallVectorImpl<ArgInfo> &Args,
+ CCAssignFn &AssignFnFixed,
+ CCAssignFn &AssignFnVarArg) const;
+
+ /// \returns True if the calling convention for a callee and its caller pass
+ /// results in the same way. Typically used for tail call eligibility checks.
+ ///
+ /// \p Info is the CallLoweringInfo for the call.
+ /// \p MF is the MachineFunction for the caller.
+ /// \p InArgs contains the results of the call.
+ /// \p CalleeAssignFnFixed is the CCAssignFn to be used for the callee for
+ /// fixed arguments.
+ /// \p CalleeAssignFnVarArg is similar, but for varargs.
+ /// \p CallerAssignFnFixed is the CCAssignFn to be used for the caller for
+ /// fixed arguments.
+ /// \p CallerAssignFnVarArg is similar, but for varargs.
+ bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF,
+ SmallVectorImpl<ArgInfo> &InArgs,
+ CCAssignFn &CalleeAssignFnFixed,
+ CCAssignFn &CalleeAssignFnVarArg,
+ CCAssignFn &CallerAssignFnFixed,
+ CCAssignFn &CallerAssignFnVarArg) const;
+
public:
CallLowering(const TargetLowering *TLI) : TLI(TLI) {}
virtual ~CallLowering() = default;
@@ -223,37 +297,10 @@ public:
/// This hook must be implemented to lower the given call instruction,
/// including argument and return value marshalling.
///
- /// \p CallConv is the calling convention to be used for the call.
- ///
- /// \p Callee is the destination of the call. It should be either a register,
- /// globaladdress, or externalsymbol.
- ///
- /// \p OrigRet is a descriptor for the return type of the function.
- ///
- /// \p OrigArgs is a list of descriptors of the arguments passed to the
- /// function.
- ///
- /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout
- /// parameter, and contains the vreg that the swifterror should be copied into
- /// after the call.
///
/// \return true if the lowering succeeded, false otherwise.
- virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv,
- const MachineOperand &Callee, const ArgInfo &OrigRet,
- ArrayRef<ArgInfo> OrigArgs,
- Register SwiftErrorVReg) const {
- if (!supportSwiftError()) {
- assert(SwiftErrorVReg == 0 && "trying to use unsupported swifterror");
- return lowerCall(MIRBuilder, CallConv, Callee, OrigRet, OrigArgs);
- }
- return false;
- }
-
- /// This hook behaves as the extended lowerCall function, but for targets that
- /// do not support swifterror value promotion.
- virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv,
- const MachineOperand &Callee, const ArgInfo &OrigRet,
- ArrayRef<ArgInfo> OrigArgs) const {
+ virtual bool lowerCall(MachineIRBuilder &MIRBuilder,
+ CallLoweringInfo &Info) const {
return false;
}
diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 0c50c9c5e0cf..4c04dc52547d 100644
--- a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -27,6 +27,8 @@ class MachineIRBuilder;
class MachineRegisterInfo;
class MachineInstr;
class MachineOperand;
+class GISelKnownBits;
+class MachineDominatorTree;
struct PreferredTuple {
LLT Ty; // The result type of the extend.
@@ -35,12 +37,17 @@ struct PreferredTuple {
};
class CombinerHelper {
+protected:
MachineIRBuilder &Builder;
MachineRegisterInfo &MRI;
GISelChangeObserver &Observer;
+ GISelKnownBits *KB;
+ MachineDominatorTree *MDT;
public:
- CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B);
+ CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B,
+ GISelKnownBits *KB = nullptr,
+ MachineDominatorTree *MDT = nullptr);
/// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes
void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const;
@@ -56,18 +63,132 @@ public:
bool matchCombineCopy(MachineInstr &MI);
void applyCombineCopy(MachineInstr &MI);
+ /// Returns true if \p DefMI precedes \p UseMI or they are the same
+ /// instruction. Both must be in the same basic block.
+ bool isPredecessor(MachineInstr &DefMI, MachineInstr &UseMI);
+
+ /// Returns true if \p DefMI dominates \p UseMI. By definition an
+ /// instruction dominates itself.
+ ///
+ /// If we haven't been provided with a MachineDominatorTree during
+ /// construction, this function returns a conservative result that tracks just
+ /// a single basic block.
+ bool dominates(MachineInstr &DefMI, MachineInstr &UseMI);
+
/// If \p MI is extend that consumes the result of a load, try to combine it.
/// Returns true if MI changed.
bool tryCombineExtendingLoads(MachineInstr &MI);
bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo);
void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo);
- bool matchCombineBr(MachineInstr &MI);
- bool tryCombineBr(MachineInstr &MI);
+ /// Combine \p MI into a pre-indexed or post-indexed load/store operation if
+ /// legal and the surrounding code makes it useful.
+ bool tryCombineIndexedLoadStore(MachineInstr &MI);
+
+ bool matchElideBrByInvertingCond(MachineInstr &MI);
+ void applyElideBrByInvertingCond(MachineInstr &MI);
+ bool tryElideBrByInvertingCond(MachineInstr &MI);
+
+ /// If \p MI is G_CONCAT_VECTORS, try to combine it.
+ /// Returns true if MI changed.
+ /// Right now, we support:
+ /// - concat_vector(undef, undef) => undef
+ /// - concat_vector(build_vector(A, B), build_vector(C, D)) =>
+ /// build_vector(A, B, C, D)
+ ///
+ /// \pre MI.getOpcode() == G_CONCAT_VECTORS.
+ bool tryCombineConcatVectors(MachineInstr &MI);
+ /// Check if the G_CONCAT_VECTORS \p MI is undef or if it
+ /// can be flattened into a build_vector.
+ /// In the first case \p IsUndef will be true.
+ /// In the second case \p Ops will contain the operands needed
+ /// to produce the flattened build_vector.
+ ///
+ /// \pre MI.getOpcode() == G_CONCAT_VECTORS.
+ bool matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef,
+ SmallVectorImpl<Register> &Ops);
+ /// Replace \p MI with a flattened build_vector with \p Ops or an
+ /// implicit_def if IsUndef is true.
+ void applyCombineConcatVectors(MachineInstr &MI, bool IsUndef,
+ const ArrayRef<Register> Ops);
+
+ /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS.
+ /// Returns true if MI changed.
+ ///
+ /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR.
+ bool tryCombineShuffleVector(MachineInstr &MI);
+ /// Check if the G_SHUFFLE_VECTOR \p MI can be replaced by a
+ /// concat_vectors.
+ /// \p Ops will contain the operands needed to produce the flattened
+ /// concat_vectors.
+ ///
+ /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR.
+ bool matchCombineShuffleVector(MachineInstr &MI,
+ SmallVectorImpl<Register> &Ops);
+ /// Replace \p MI with a concat_vectors with \p Ops.
+ void applyCombineShuffleVector(MachineInstr &MI,
+ const ArrayRef<Register> Ops);
+
+ /// Optimize memcpy intrinsics et al, e.g. constant len calls.
+ /// /p MaxLen if non-zero specifies the max length of a mem libcall to inline.
+ ///
+ /// For example (pre-indexed):
+ ///
+ /// $addr = G_GEP $base, $offset
+ /// [...]
+ /// $val = G_LOAD $addr
+ /// [...]
+ /// $whatever = COPY $addr
+ ///
+ /// -->
+ ///
+ /// $val, $addr = G_INDEXED_LOAD $base, $offset, 1 (IsPre)
+ /// [...]
+ /// $whatever = COPY $addr
+ ///
+ /// or (post-indexed):
+ ///
+ /// G_STORE $val, $base
+ /// [...]
+ /// $addr = G_GEP $base, $offset
+ /// [...]
+ /// $whatever = COPY $addr
+ ///
+ /// -->
+ ///
+ /// $addr = G_INDEXED_STORE $val, $base, $offset
+ /// [...]
+ /// $whatever = COPY $addr
+ bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0);
/// Try to transform \p MI by using all of the above
/// combine functions. Returns true if changed.
bool tryCombine(MachineInstr &MI);
+
+private:
+ // Memcpy family optimization helpers.
+ bool optimizeMemcpy(MachineInstr &MI, Register Dst, Register Src,
+ unsigned KnownLen, unsigned DstAlign, unsigned SrcAlign,
+ bool IsVolatile);
+ bool optimizeMemmove(MachineInstr &MI, Register Dst, Register Src,
+ unsigned KnownLen, unsigned DstAlign, unsigned SrcAlign,
+ bool IsVolatile);
+ bool optimizeMemset(MachineInstr &MI, Register Dst, Register Val,
+ unsigned KnownLen, unsigned DstAlign, bool IsVolatile);
+
+ /// Given a non-indexed load or store instruction \p MI, find an offset that
+ /// can be usefully and legally folded into it as a post-indexing operation.
+ ///
+ /// \returns true if a candidate is found.
+ bool findPostIndexCandidate(MachineInstr &MI, Register &Addr, Register &Base,
+ Register &Offset);
+
+ /// Given a non-indexed load or store instruction \p MI, find an offset that
+ /// can be usefully and legally folded into it as a pre-indexing operation.
+ ///
+ /// \returns true if a candidate is found.
+ bool findPreIndexCandidate(MachineInstr &MI, Register &Addr, Register &Base,
+ Register &Offset);
};
} // namespace llvm
diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
index 3b09a8e2b479..ad645a46bbe6 100644
--- a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
+++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
@@ -27,9 +27,11 @@ class MachineRegisterInfo;
class CombinerInfo {
public:
CombinerInfo(bool AllowIllegalOps, bool ShouldLegalizeIllegal,
- LegalizerInfo *LInfo)
+ LegalizerInfo *LInfo, bool OptEnabled, bool OptSize,
+ bool MinSize)
: IllegalOpsAllowed(AllowIllegalOps),
- LegalizeIllegalOps(ShouldLegalizeIllegal), LInfo(LInfo) {
+ LegalizeIllegalOps(ShouldLegalizeIllegal), LInfo(LInfo),
+ EnableOpt(OptEnabled), EnableOptSize(OptSize), EnableMinSize(MinSize) {
assert(((AllowIllegalOps || !LegalizeIllegalOps) || LInfo) &&
"Expecting legalizerInfo when illegalops not allowed");
}
@@ -43,6 +45,15 @@ public:
bool LegalizeIllegalOps; // TODO: Make use of this.
const LegalizerInfo *LInfo;
+ /// Whether optimizations should be enabled. This is to distinguish between
+ /// uses of the combiner unconditionally and only when optimizations are
+ /// specifically enabled/
+ bool EnableOpt;
+ /// Whether we're optimizing for size.
+ bool EnableOptSize;
+ /// Whether we're optimizing for minsize (-Oz).
+ bool EnableMinSize;
+
/// Attempt to combine instructions using MI as the root.
///
/// Use Observer to report the creation, modification, and erasure of
diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h
index e817d9b4550e..df196bfbd437 100644
--- a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h
+++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h
@@ -54,6 +54,17 @@ public:
return buildConstant(Dst, MaybeCst->getSExtValue());
break;
}
+ case TargetOpcode::G_SEXT_INREG: {
+ assert(DstOps.size() == 1 && "Invalid dst ops");
+ assert(SrcOps.size() == 2 && "Invalid src ops");
+ const DstOp &Dst = DstOps[0];
+ const SrcOp &Src0 = SrcOps[0];
+ const SrcOp &Src1 = SrcOps[1];
+ if (auto MaybeCst =
+ ConstantFoldExtOp(Opc, Src0.getReg(), Src1.getImm(), *getMRI()))
+ return buildConstant(Dst, MaybeCst->getSExtValue());
+ break;
+ }
}
return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps);
}
diff --git a/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h b/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
new file mode 100644
index 000000000000..dfe5a7f3177d
--- /dev/null
+++ b/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
@@ -0,0 +1,111 @@
+//===- llvm/CodeGen/GlobalISel/GISelKnownBits.h ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Provides analysis for querying information about KnownBits during GISel
+/// passes.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H
+#define LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H
+
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/KnownBits.h"
+
+namespace llvm {
+
+class TargetLowering;
+class DataLayout;
+
+class GISelKnownBits : public GISelChangeObserver {
+ MachineFunction &MF;
+ MachineRegisterInfo &MRI;
+ const TargetLowering &TL;
+ const DataLayout &DL;
+
+public:
+ GISelKnownBits(MachineFunction &MF);
+ virtual ~GISelKnownBits() = default;
+ void setMF(MachineFunction &MF);
+ virtual void computeKnownBitsImpl(Register R, KnownBits &Known,
+ const APInt &DemandedElts,
+ unsigned Depth = 0);
+
+ // KnownBitsAPI
+ KnownBits getKnownBits(Register R);
+ // Calls getKnownBits for first operand def of MI.
+ KnownBits getKnownBits(MachineInstr &MI);
+ APInt getKnownZeroes(Register R);
+ APInt getKnownOnes(Register R);
+
+ /// \return true if 'V & Mask' is known to be zero in DemandedElts. We use
+ /// this predicate to simplify operations downstream.
+ /// Mask is known to be zero for bits that V cannot have.
+ bool maskedValueIsZero(Register Val, const APInt &Mask) {
+ return Mask.isSubsetOf(getKnownBits(Val).Zero);
+ }
+
+ /// \return true if the sign bit of Op is known to be zero. We use this
+ /// predicate to simplify operations downstream.
+ bool signBitIsZero(Register Op);
+
+ // FIXME: Is this the right place for G_FRAME_INDEX? Should it be in
+ // TargetLowering?
+ void computeKnownBitsForFrameIndex(Register R, KnownBits &Known,
+ const APInt &DemandedElts,
+ unsigned Depth = 0);
+ static Align inferAlignmentForFrameIdx(int FrameIdx, int Offset,
+ const MachineFunction &MF);
+ static void computeKnownBitsForAlignment(KnownBits &Known,
+ MaybeAlign Alignment);
+
+ // Try to infer alignment for MI.
+ static MaybeAlign inferPtrAlignment(const MachineInstr &MI);
+
+ // Observer API. No-op for non-caching implementation.
+ void erasingInstr(MachineInstr &MI) override{};
+ void createdInstr(MachineInstr &MI) override{};
+ void changingInstr(MachineInstr &MI) override{};
+ void changedInstr(MachineInstr &MI) override{};
+
+protected:
+ unsigned getMaxDepth() const { return 6; }
+};
+
+/// To use KnownBitsInfo analysis in a pass,
+/// KnownBitsInfo &Info = getAnalysis<GISelKnownBitsInfoAnalysis>().get(MF);
+/// Add to observer if the Info is caching.
+/// WrapperObserver.addObserver(Info);
+
+/// Eventually add other features such as caching/ser/deserializing
+/// to MIR etc. Those implementations can derive from GISelKnownBits
+/// and override computeKnownBitsImpl.
+class GISelKnownBitsAnalysis : public MachineFunctionPass {
+ std::unique_ptr<GISelKnownBits> Info;
+
+public:
+ static char ID;
+ GISelKnownBitsAnalysis() : MachineFunctionPass(ID) {
+ initializeGISelKnownBitsAnalysisPass(*PassRegistry::getPassRegistry());
+ }
+ GISelKnownBits &get(MachineFunction &MF) {
+ if (!Info)
+ Info = std::make_unique<GISelKnownBits>(MF);
+ return *Info.get();
+ }
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ void releaseMemory() override { Info.reset(); }
+};
+} // namespace llvm
+
+#endif // ifdef
diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 8654ba83f08d..bdb92aa4689d 100644
--- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -213,8 +213,8 @@ private:
bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
/// Translate an LLVM string intrinsic (memcpy, memset, ...).
- bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
- unsigned ID);
+ bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
+ Intrinsic::ID ID);
void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
@@ -243,6 +243,10 @@ private:
bool valueIsSplit(const Value &V,
SmallVectorImpl<uint64_t> *Offsets = nullptr);
+ /// Common code for translating normal calls or invokes.
+ bool translateCallSite(const ImmutableCallSite &CS,
+ MachineIRBuilder &MIRBuilder);
+
/// Translate call instruction.
/// \pre \p U is a call instruction.
bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);
@@ -514,6 +518,10 @@ private:
// function has the optnone attribute.
bool EnableOpts = false;
+ /// True when the block contains a tail call. This allows the IRTranslator to
+ /// stop translating such blocks early.
+ bool HasTailCall = false;
+
/// Switch analysis and optimization.
class GISelSwitchLowering : public SwitchCG::SwitchLowering {
public:
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
index e9b93be76754..fd3dc743000b 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
@@ -31,6 +31,7 @@ namespace llvm {
class APInt;
class APFloat;
+class GISelKnownBits;
class MachineInstr;
class MachineInstrBuilder;
class MachineFunction;
@@ -148,6 +149,13 @@ enum {
/// - AddrSpaceN+1 ...
GIM_CheckMemoryAddressSpace,
+ /// Check the minimum alignment of the memory access for the given machine
+ /// memory operand.
+ /// - InsnID - Instruction ID
+ /// - MMOIdx - MMO index
+ /// - MinAlign - Minimum acceptable alignment
+ GIM_CheckMemoryAlignment,
+
/// Check the size of the memory access for the given machine memory operand
/// against the size of an operand.
/// - InsnID - Instruction ID
@@ -201,11 +209,22 @@ enum {
/// - Expected Intrinsic ID
GIM_CheckIntrinsicID,
+ /// Check the operand is a specific predicate
+ /// - InsnID - Instruction ID
+ /// - OpIdx - Operand index
+ /// - Expected predicate
+ GIM_CheckCmpPredicate,
+
/// Check the specified operand is an MBB
/// - InsnID - Instruction ID
/// - OpIdx - Operand index
GIM_CheckIsMBB,
+ /// Check the specified operand is an Imm
+ /// - InsnID - Instruction ID
+ /// - OpIdx - Operand index
+ GIM_CheckIsImm,
+
/// Check if the specified operand is safe to fold into the current
/// instruction.
/// - InsnID - Instruction ID
@@ -365,7 +384,20 @@ public:
/// if returns true:
/// for I in all mutated/inserted instructions:
/// !isPreISelGenericOpcode(I.getOpcode())
- virtual bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const = 0;
+ virtual bool select(MachineInstr &I) = 0;
+
+ CodeGenCoverage *CoverageInfo = nullptr;
+ GISelKnownBits *KnownBits = nullptr;
+ MachineFunction *MF = nullptr;
+
+ /// Setup per-MF selector state.
+ virtual void setupMF(MachineFunction &mf,
+ GISelKnownBits &KB,
+ CodeGenCoverage &covinfo) {
+ CoverageInfo = &covinfo;
+ KnownBits = &KB;
+ MF = &mf;
+ }
protected:
using ComplexRendererFns =
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
index e8ee4af0cb0b..08f2f54bcf90 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -98,7 +98,7 @@ bool InstructionSelector::executeMatchTable(
return false;
break;
}
- if (TRI.isPhysicalRegister(MO.getReg())) {
+ if (Register::isPhysicalRegister(MO.getReg())) {
DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
dbgs() << CurrentIdx << ": Is a physical register\n");
if (handleReject() == RejectAndGiveUp)
@@ -409,6 +409,30 @@ bool InstructionSelector::executeMatchTable(
return false;
break;
}
+ case GIM_CheckMemoryAlignment: {
+ int64_t InsnID = MatchTable[CurrentIdx++];
+ int64_t MMOIdx = MatchTable[CurrentIdx++];
+ unsigned MinAlign = MatchTable[CurrentIdx++];
+
+ assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+
+ if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
+ if (handleReject() == RejectAndGiveUp)
+ return false;
+ break;
+ }
+
+ MachineMemOperand *MMO
+ = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
+ DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
+ dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
+ << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
+ << ")->getAlignment() >= " << MinAlign << ")\n");
+ if (MMO->getAlignment() < MinAlign && handleReject() == RejectAndGiveUp)
+ return false;
+
+ break;
+ }
case GIM_CheckMemorySizeEqualTo: {
int64_t InsnID = MatchTable[CurrentIdx++];
int64_t MMOIdx = MatchTable[CurrentIdx++];
@@ -638,7 +662,21 @@ bool InstructionSelector::executeMatchTable(
return false;
break;
}
-
+ case GIM_CheckCmpPredicate: {
+ int64_t InsnID = MatchTable[CurrentIdx++];
+ int64_t OpIdx = MatchTable[CurrentIdx++];
+ int64_t Value = MatchTable[CurrentIdx++];
+ DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
+ dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
+ << InsnID << "]->getOperand(" << OpIdx
+ << "), Value=" << Value << ")\n");
+ assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+ MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
+ if (!MO.isPredicate() || MO.getPredicate() != Value)
+ if (handleReject() == RejectAndGiveUp)
+ return false;
+ break;
+ }
case GIM_CheckIsMBB: {
int64_t InsnID = MatchTable[CurrentIdx++];
int64_t OpIdx = MatchTable[CurrentIdx++];
@@ -652,7 +690,19 @@ bool InstructionSelector::executeMatchTable(
}
break;
}
-
+ case GIM_CheckIsImm: {
+ int64_t InsnID = MatchTable[CurrentIdx++];
+ int64_t OpIdx = MatchTable[CurrentIdx++];
+ DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
+ dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
+ << "]->getOperand(" << OpIdx << "))\n");
+ assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+ if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
+ if (handleReject() == RejectAndGiveUp)
+ return false;
+ }
+ break;
+ }
case GIM_CheckIsSafeToFold: {
int64_t InsnID = MatchTable[CurrentIdx++];
DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
@@ -792,11 +842,13 @@ bool InstructionSelector::executeMatchTable(
case GIR_AddRegister: {
int64_t InsnID = MatchTable[CurrentIdx++];
int64_t RegNum = MatchTable[CurrentIdx++];
+ uint64_t RegFlags = MatchTable[CurrentIdx++];
assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
- OutMIs[InsnID].addReg(RegNum);
- DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
- dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
- << InsnID << "], " << RegNum << ")\n");
+ OutMIs[InsnID].addReg(RegNum, RegFlags);
+ DEBUG_WITH_TYPE(
+ TgtInstructionSelector::getName(),
+ dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
+ << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
break;
}
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index a22778b8848c..7f960e727846 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -47,8 +47,7 @@ public:
bool tryCombineAnyExt(MachineInstr &MI,
SmallVectorImpl<MachineInstr *> &DeadInsts) {
- if (MI.getOpcode() != TargetOpcode::G_ANYEXT)
- return false;
+ assert(MI.getOpcode() == TargetOpcode::G_ANYEXT);
Builder.setInstr(MI);
Register DstReg = MI.getOperand(0).getReg();
@@ -93,9 +92,7 @@ public:
bool tryCombineZExt(MachineInstr &MI,
SmallVectorImpl<MachineInstr *> &DeadInsts) {
-
- if (MI.getOpcode() != TargetOpcode::G_ZEXT)
- return false;
+ assert(MI.getOpcode() == TargetOpcode::G_ZEXT);
Builder.setInstr(MI);
Register DstReg = MI.getOperand(0).getReg();
@@ -136,32 +133,24 @@ public:
bool tryCombineSExt(MachineInstr &MI,
SmallVectorImpl<MachineInstr *> &DeadInsts) {
-
- if (MI.getOpcode() != TargetOpcode::G_SEXT)
- return false;
+ assert(MI.getOpcode() == TargetOpcode::G_SEXT);
Builder.setInstr(MI);
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
- // sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c
+ // sext(trunc x) - > (sext_inreg (aext/copy/trunc x), c)
Register TruncSrc;
if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
LLT DstTy = MRI.getType(DstReg);
- // Guess on the RHS shift amount type, which should be re-legalized if
- // applicable.
- if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy, DstTy}}) ||
- isInstUnsupported({TargetOpcode::G_ASHR, {DstTy, DstTy}}) ||
- isConstantUnsupported(DstTy))
+ if (isInstUnsupported({TargetOpcode::G_SEXT_INREG, {DstTy}}))
return false;
LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
LLT SrcTy = MRI.getType(SrcReg);
- unsigned ShAmt = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits();
- auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt);
- auto MIBShl = Builder.buildInstr(
- TargetOpcode::G_SHL, {DstTy},
- {Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBShAmt});
- Builder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {MIBShl, MIBShAmt});
+ uint64_t SizeInBits = SrcTy.getScalarSizeInBits();
+ Builder.buildInstr(
+ TargetOpcode::G_SEXT_INREG, {DstReg},
+ {Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), SizeInBits});
markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
return true;
}
@@ -172,9 +161,8 @@ public:
bool tryFoldImplicitDef(MachineInstr &MI,
SmallVectorImpl<MachineInstr *> &DeadInsts) {
unsigned Opcode = MI.getOpcode();
- if (Opcode != TargetOpcode::G_ANYEXT && Opcode != TargetOpcode::G_ZEXT &&
- Opcode != TargetOpcode::G_SEXT)
- return false;
+ assert(Opcode == TargetOpcode::G_ANYEXT || Opcode == TargetOpcode::G_ZEXT ||
+ Opcode == TargetOpcode::G_SEXT);
if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(1).getReg(), MRI)) {
@@ -203,21 +191,38 @@ public:
return false;
}
- static unsigned getMergeOpcode(LLT OpTy, LLT DestTy) {
+ static unsigned canFoldMergeOpcode(unsigned MergeOp, unsigned ConvertOp,
+ LLT OpTy, LLT DestTy) {
if (OpTy.isVector() && DestTy.isVector())
- return TargetOpcode::G_CONCAT_VECTORS;
+ return MergeOp == TargetOpcode::G_CONCAT_VECTORS;
+
+ if (OpTy.isVector() && !DestTy.isVector()) {
+ if (MergeOp == TargetOpcode::G_BUILD_VECTOR)
+ return true;
- if (OpTy.isVector() && !DestTy.isVector())
- return TargetOpcode::G_BUILD_VECTOR;
+ if (MergeOp == TargetOpcode::G_CONCAT_VECTORS) {
+ if (ConvertOp == 0)
+ return true;
- return TargetOpcode::G_MERGE_VALUES;
+ const unsigned OpEltSize = OpTy.getElementType().getSizeInBits();
+
+ // Don't handle scalarization with a cast that isn't in the same
+ // direction as the vector cast. This could be handled, but it would
+ // require more intermediate unmerges.
+ if (ConvertOp == TargetOpcode::G_TRUNC)
+ return DestTy.getSizeInBits() <= OpEltSize;
+ return DestTy.getSizeInBits() >= OpEltSize;
+ }
+
+ return false;
+ }
+
+ return MergeOp == TargetOpcode::G_MERGE_VALUES;
}
bool tryCombineMerges(MachineInstr &MI,
SmallVectorImpl<MachineInstr *> &DeadInsts) {
-
- if (MI.getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
- return false;
+ assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
unsigned NumDefs = MI.getNumOperands() - 1;
MachineInstr *SrcDef =
@@ -237,16 +242,14 @@ public:
MergeI = getDefIgnoringCopies(SrcDef->getOperand(1).getReg(), MRI);
}
- // FIXME: Handle scalarizing concat_vectors (scalar result type with vector
- // source)
- unsigned MergingOpcode = getMergeOpcode(OpTy, DestTy);
- if (!MergeI || MergeI->getOpcode() != MergingOpcode)
+ if (!MergeI || !canFoldMergeOpcode(MergeI->getOpcode(),
+ ConvertOp, OpTy, DestTy))
return false;
const unsigned NumMergeRegs = MergeI->getNumOperands() - 1;
if (NumMergeRegs < NumDefs) {
- if (ConvertOp != 0 || NumDefs % NumMergeRegs != 0)
+ if (NumDefs % NumMergeRegs != 0)
return false;
Builder.setInstr(MI);
@@ -264,7 +267,22 @@ public:
++j, ++DefIdx)
DstRegs.push_back(MI.getOperand(DefIdx).getReg());
- Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg());
+ if (ConvertOp) {
+ SmallVector<Register, 2> TmpRegs;
+ // This is a vector that is being scalarized and casted. Extract to
+ // the element type, and do the conversion on the scalars.
+ LLT MergeEltTy
+ = MRI.getType(MergeI->getOperand(0).getReg()).getElementType();
+ for (unsigned j = 0; j < NumMergeRegs; ++j)
+ TmpRegs.push_back(MRI.createGenericVirtualRegister(MergeEltTy));
+
+ Builder.buildUnmerge(TmpRegs, MergeI->getOperand(Idx + 1).getReg());
+
+ for (unsigned j = 0; j < NumMergeRegs; ++j)
+ Builder.buildInstr(ConvertOp, {DstRegs[j]}, {TmpRegs[j]});
+ } else {
+ Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg());
+ }
}
} else if (NumMergeRegs > NumDefs) {
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index a0f21e8b19d7..fbfe71255a38 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -200,6 +200,13 @@ public:
LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
LLT MoreTy);
+ LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI,
+ unsigned TypeIdx,
+ LLT NarrowTy);
+ LegalizeResult fewerElementsVectorBuildVector(MachineInstr &MI,
+ unsigned TypeIdx,
+ LLT NarrowTy);
+
LegalizeResult
reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
@@ -219,9 +226,17 @@ public:
LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI);
LegalizeResult lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI);
+ LegalizeResult lowerFMad(MachineInstr &MI);
+ LegalizeResult lowerUnmergeValues(MachineInstr &MI);
+ LegalizeResult lowerShuffleVector(MachineInstr &MI);
+ LegalizeResult lowerDynStackAlloc(MachineInstr &MI);
+ LegalizeResult lowerExtract(MachineInstr &MI);
+ LegalizeResult lowerInsert(MachineInstr &MI);
+ LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI);
private:
MachineRegisterInfo &MRI;
@@ -236,6 +251,11 @@ createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
const CallLowering::ArgInfo &Result,
ArrayRef<CallLowering::ArgInfo> Args);
+/// Create a libcall to memcpy et al.
+LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI,
+ MachineInstr &MI);
+
} // End namespace llvm.
#endif
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 513c98f2d23f..1cf62d1fde59 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -331,6 +331,8 @@ class LegalizeRuleSet {
/// individually handled.
SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC -
MCOI::OPERAND_FIRST_GENERIC + 2};
+ SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM -
+ MCOI::OPERAND_FIRST_GENERIC_IMM + 2};
#endif
unsigned typeIdx(unsigned TypeIdx) {
@@ -342,9 +344,21 @@ class LegalizeRuleSet {
#endif
return TypeIdx;
}
- void markAllTypeIdxsAsCovered() {
+
+ unsigned immIdx(unsigned ImmIdx) {
+ assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM -
+ MCOI::OPERAND_FIRST_GENERIC_IMM) &&
+ "Imm Index is out of bounds");
+#ifndef NDEBUG
+ ImmIdxsCovered.set(ImmIdx);
+#endif
+ return ImmIdx;
+ }
+
+ void markAllIdxsAsCovered() {
#ifndef NDEBUG
TypeIdxsCovered.set();
+ ImmIdxsCovered.set();
#endif
}
@@ -403,6 +417,15 @@ class LegalizeRuleSet {
return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types),
Mutation);
}
+ /// Use the given action when type index 0 is any type in the given list and
+ /// imm index 0 is anything. Action should not be an action that requires
+ /// mutation.
+ LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action,
+ std::initializer_list<LLT> Types) {
+ using namespace LegalityPredicates;
+ immIdx(0); // Inform verifier imm idx 0 is handled.
+ return actionIf(Action, typeInSet(typeIdx(0), Types));
+ }
/// Use the given action when type indexes 0 and 1 are both in the given list.
/// That is, the type pair is in the cartesian product of the list.
/// Action should not be an action that requires mutation.
@@ -454,7 +477,7 @@ public:
LegalizeRuleSet &legalIf(LegalityPredicate Predicate) {
// We have no choice but conservatively assume that the free-form
// user-provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Legal, Predicate);
}
/// The instruction is legal when type index 0 is any type in the given list.
@@ -466,6 +489,12 @@ public:
LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
return actionFor(LegalizeAction::Legal, Types);
}
+ /// The instruction is legal when type index 0 is any type in the given list
+ /// and imm index 0 is anything.
+ LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
+ markAllIdxsAsCovered();
+ return actionForTypeWithAnyImm(LegalizeAction::Legal, Types);
+ }
/// The instruction is legal when type indexes 0 and 1 along with the memory
/// size and minimum alignment is any type and size tuple in the given list.
LegalizeRuleSet &legalForTypesWithMemDesc(
@@ -497,7 +526,7 @@ public:
LegalizeRuleSet &alwaysLegal() {
using namespace LegalizeMutations;
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Legal, always);
}
@@ -506,7 +535,7 @@ public:
using namespace LegalizeMutations;
// We have no choice but conservatively assume that predicate-less lowering
// properly handles all type indices by design:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Lower, always);
}
/// The instruction is lowered if predicate is true. Keep type index 0 as the
@@ -515,7 +544,7 @@ public:
using namespace LegalizeMutations;
// We have no choice but conservatively assume that lowering with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Lower, Predicate);
}
/// The instruction is lowered if predicate is true.
@@ -523,7 +552,7 @@ public:
LegalizeMutation Mutation) {
// We have no choice but conservatively assume that lowering with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Lower, Predicate, Mutation);
}
/// The instruction is lowered when type index 0 is any type in the given
@@ -571,7 +600,7 @@ public:
LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) {
// We have no choice but conservatively assume that a libcall with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Libcall, Predicate);
}
LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) {
@@ -597,7 +626,7 @@ public:
LegalizeMutation Mutation) {
// We have no choice but conservatively assume that an action with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation);
}
/// Narrow the scalar to the one selected by the mutation if the predicate is
@@ -606,7 +635,7 @@ public:
LegalizeMutation Mutation) {
// We have no choice but conservatively assume that an action with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation);
}
@@ -616,7 +645,7 @@ public:
LegalizeMutation Mutation) {
// We have no choice but conservatively assume that an action with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::MoreElements, Predicate, Mutation);
}
/// Remove elements to reach the type selected by the mutation if the
@@ -625,7 +654,7 @@ public:
LegalizeMutation Mutation) {
// We have no choice but conservatively assume that an action with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::FewerElements, Predicate, Mutation);
}
@@ -640,11 +669,15 @@ public:
return actionIf(LegalizeAction::Unsupported,
LegalityPredicates::memSizeInBytesNotPow2(0));
}
+ LegalizeRuleSet &lowerIfMemSizeNotPow2() {
+ return actionIf(LegalizeAction::Lower,
+ LegalityPredicates::memSizeInBytesNotPow2(0));
+ }
LegalizeRuleSet &customIf(LegalityPredicate Predicate) {
// We have no choice but conservatively assume that a custom action with a
// free-form user provided Predicate properly handles all type indices:
- markAllTypeIdxsAsCovered();
+ markAllIdxsAsCovered();
return actionIf(LegalizeAction::Custom, Predicate);
}
LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
@@ -882,6 +915,10 @@ public:
/// LegalizeRuleSet in any way at all.
/// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const;
+ /// Check if there is no imm index which is obviously not handled by the
+ /// LegalizeRuleSet in any way at all.
+ /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
+ bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const;
/// Apply the ruleset to the given LegalityQuery.
LegalizeActionStep apply(const LegalityQuery &Query) const;
diff --git a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index 13eddd9539fa..be12341f5763 100644
--- a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -21,7 +21,7 @@ namespace llvm {
namespace MIPatternMatch {
template <typename Reg, typename Pattern>
-bool mi_match(Reg R, MachineRegisterInfo &MRI, Pattern &&P) {
+bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P) {
return P.match(MRI, R);
}
@@ -30,7 +30,7 @@ template <typename SubPatternT> struct OneUse_match {
SubPatternT SubPat;
OneUse_match(const SubPatternT &SP) : SubPat(SP) {}
- bool match(MachineRegisterInfo &MRI, unsigned Reg) {
+ bool match(const MachineRegisterInfo &MRI, unsigned Reg) {
return MRI.hasOneUse(Reg) && SubPat.match(MRI, Reg);
}
};
@@ -71,7 +71,7 @@ inline operand_type_match m_Reg() { return operand_type_match(); }
/// Matching combinators.
template <typename... Preds> struct And {
template <typename MatchSrc>
- bool match(MachineRegisterInfo &MRI, MatchSrc &&src) {
+ bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) {
return true;
}
};
@@ -83,14 +83,14 @@ struct And<Pred, Preds...> : And<Preds...> {
: And<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {
}
template <typename MatchSrc>
- bool match(MachineRegisterInfo &MRI, MatchSrc &&src) {
+ bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) {
return P.match(MRI, src) && And<Preds...>::match(MRI, src);
}
};
template <typename... Preds> struct Or {
template <typename MatchSrc>
- bool match(MachineRegisterInfo &MRI, MatchSrc &&src) {
+ bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) {
return false;
}
};
@@ -101,7 +101,7 @@ struct Or<Pred, Preds...> : Or<Preds...> {
Or(Pred &&p, Preds &&... preds)
: Or<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {}
template <typename MatchSrc>
- bool match(MachineRegisterInfo &MRI, MatchSrc &&src) {
+ bool match(const MachineRegisterInfo &MRI, MatchSrc &&src) {
return P.match(MRI, src) || Or<Preds...>::match(MRI, src);
}
};
@@ -175,7 +175,8 @@ struct BinaryOp_match {
RHS_P R;
BinaryOp_match(const LHS_P &LHS, const RHS_P &RHS) : L(LHS), R(RHS) {}
- template <typename OpTy> bool match(MachineRegisterInfo &MRI, OpTy &&Op) {
+ template <typename OpTy>
+ bool match(const MachineRegisterInfo &MRI, OpTy &&Op) {
MachineInstr *TmpMI;
if (mi_match(Op, MRI, m_MInstr(TmpMI))) {
if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 3) {
@@ -242,7 +243,8 @@ template <typename SrcTy, unsigned Opcode> struct UnaryOp_match {
SrcTy L;
UnaryOp_match(const SrcTy &LHS) : L(LHS) {}
- template <typename OpTy> bool match(MachineRegisterInfo &MRI, OpTy &&Op) {
+ template <typename OpTy>
+ bool match(const MachineRegisterInfo &MRI, OpTy &&Op) {
MachineInstr *TmpMI;
if (mi_match(Op, MRI, m_MInstr(TmpMI))) {
if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 2) {
@@ -323,7 +325,7 @@ struct CheckType {
LLT Ty;
CheckType(const LLT &Ty) : Ty(Ty) {}
- bool match(MachineRegisterInfo &MRI, unsigned Reg) {
+ bool match(const MachineRegisterInfo &MRI, unsigned Reg) {
return MRI.getType(Reg) == Ty;
}
};
diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 10d712176b1b..416f9c19f794 100644
--- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -122,14 +122,22 @@ class SrcOp {
MachineInstrBuilder SrcMIB;
Register Reg;
CmpInst::Predicate Pred;
+ int64_t Imm;
};
public:
- enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate };
+ enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate, Ty_Imm };
SrcOp(Register R) : Reg(R), Ty(SrcType::Ty_Reg) {}
SrcOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(SrcType::Ty_Reg) {}
SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {}
SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {}
+ /// Use of registers held in unsigned integer variables (or more rarely signed
+ /// integers) is no longer permitted to avoid ambiguity with upcoming support
+ /// for immediates.
+ SrcOp(unsigned) = delete;
+ SrcOp(int) = delete;
+ SrcOp(uint64_t V) : Imm(V), Ty(SrcType::Ty_Imm) {}
+ SrcOp(int64_t V) : Imm(V), Ty(SrcType::Ty_Imm) {}
void addSrcToMIB(MachineInstrBuilder &MIB) const {
switch (Ty) {
@@ -142,12 +150,16 @@ public:
case SrcType::Ty_MIB:
MIB.addUse(SrcMIB->getOperand(0).getReg());
break;
+ case SrcType::Ty_Imm:
+ MIB.addImm(Imm);
+ break;
}
}
LLT getLLTTy(const MachineRegisterInfo &MRI) const {
switch (Ty) {
case SrcType::Ty_Predicate:
+ case SrcType::Ty_Imm:
llvm_unreachable("Not a register operand");
case SrcType::Ty_Reg:
return MRI.getType(Reg);
@@ -160,6 +172,7 @@ public:
Register getReg() const {
switch (Ty) {
case SrcType::Ty_Predicate:
+ case SrcType::Ty_Imm:
llvm_unreachable("Not a register operand");
case SrcType::Ty_Reg:
return Reg;
@@ -178,6 +191,15 @@ public:
}
}
+ int64_t getImm() const {
+ switch (Ty) {
+ case SrcType::Ty_Imm:
+ return Imm;
+ default:
+ llvm_unreachable("Not an immediate");
+ }
+ }
+
SrcType getSrcOpKind() const { return Ty; }
private:
@@ -348,6 +370,17 @@ public:
/// given. Convert "llvm.dbg.label Label" to "DBG_LABEL Label".
MachineInstrBuilder buildDbgLabel(const MDNode *Label);
+ /// Build and insert \p Res = G_DYN_STACKALLOC \p Size, \p Align
+ ///
+ /// G_DYN_STACKALLOC does a dynamic stack allocation and writes the address of
+ /// the allocated memory into \p Res.
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p Res must be a generic virtual register with pointer type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size,
+ unsigned Align);
+
/// Build and insert \p Res = G_FRAME_INDEX \p Idx
///
/// G_FRAME_INDEX materializes the address of an alloca value or other
@@ -489,11 +522,21 @@ public:
return buildInstr(TargetOpcode::G_PTRTOINT, {Dst}, {Src});
}
+ /// Build and insert a G_INTTOPTR instruction.
+ MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src) {
+ return buildInstr(TargetOpcode::G_INTTOPTR, {Dst}, {Src});
+ }
+
/// Build and insert \p Dst = G_BITCAST \p Src
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src) {
return buildInstr(TargetOpcode::G_BITCAST, {Dst}, {Src});
}
+ /// Build and insert \p Dst = G_ADDRSPACE_CAST \p Src
+ MachineInstrBuilder buildAddrSpaceCast(const DstOp &Dst, const SrcOp &Src) {
+ return buildInstr(TargetOpcode::G_ADDRSPACE_CAST, {Dst}, {Src});
+ }
+
/// \return The opcode of the extension the target wants to use for boolean
/// values.
unsigned getBoolExtOp(bool IsVec, bool IsFP) const;
@@ -867,7 +910,8 @@ public:
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res,
- const SrcOp &Op0, const SrcOp &Op1);
+ const SrcOp &Op0, const SrcOp &Op1,
+ Optional<unsigned> Flags = None);
/// Build and insert a \p Res = G_SELECT \p Tst, \p Op0, \p Op1
///
@@ -880,7 +924,8 @@ public:
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst,
- const SrcOp &Op0, const SrcOp &Op1);
+ const SrcOp &Op0, const SrcOp &Op1,
+ Optional<unsigned> Flags = None);
/// Build and insert \p Res = G_INSERT_VECTOR_ELT \p Val,
/// \p Elt, \p Idx
@@ -961,8 +1006,8 @@ public:
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes,
- Register Addr, Register Val,
+ MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes,
+ const SrcOp &Addr, const SrcOp &Val,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`.
@@ -1135,6 +1180,16 @@ public:
MachineInstrBuilder buildAtomicRMWUmin(Register OldValRes, Register Addr,
Register Val, MachineMemOperand &MMO);
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_FADD Addr, Val, MMO`.
+ MachineInstrBuilder buildAtomicRMWFAdd(
+ const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
+ MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_FSUB Addr, Val, MMO`.
+ MachineInstrBuilder buildAtomicRMWFSub(
+ const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
+ MachineMemOperand &MMO);
+
/// Build and insert `G_FENCE Ordering, Scope`.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope);
@@ -1210,6 +1265,12 @@ public:
return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags);
}
+ MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1,
+ Optional<unsigned> Flags = None) {
+ return buildInstr(TargetOpcode::G_FMUL, {Dst}, {Src0, Src1}, Flags);
+ }
+
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0,
const SrcOp &Src1,
Optional<unsigned> Flags = None) {
@@ -1300,8 +1361,9 @@ public:
/// Build and insert \p Res = G_FADD \p Op0, \p Op1
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0,
- const SrcOp &Src1) {
- return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1});
+ const SrcOp &Src1,
+ Optional<unsigned> Flags = None) {
+ return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1}, Flags);
}
/// Build and insert \p Res = G_FSUB \p Op0, \p Op1
@@ -1316,14 +1378,23 @@ public:
return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2});
}
+ /// Build and insert \p Res = G_FMAD \p Op0, \p Op1, \p Op2
+ MachineInstrBuilder buildFMAD(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1, const SrcOp &Src2,
+ Optional<unsigned> Flags = None) {
+ return buildInstr(TargetOpcode::G_FMAD, {Dst}, {Src0, Src1, Src2}, Flags);
+ }
+
/// Build and insert \p Res = G_FNEG \p Op0
- MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0) {
- return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0});
+ MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0,
+ Optional<unsigned> Flags = None) {
+ return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0}, Flags);
}
/// Build and insert \p Res = G_FABS \p Op0
- MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0) {
- return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0});
+ MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0,
+ Optional<unsigned> Flags = None) {
+ return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0}, Flags);
}
/// Build and insert \p Dst = G_FCANONICALIZE \p Src0
diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h
index 4cdaa48fb689..8af2853473c2 100644
--- a/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -16,6 +16,8 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/Register.h"
+#include "llvm/Support/LowLevelTypeImpl.h"
+#include "llvm/Support/MachineValueType.h"
namespace llvm {
@@ -117,14 +119,16 @@ struct ValueAndVReg {
unsigned VReg;
};
/// If \p VReg is defined by a statically evaluable chain of
-/// instructions rooted on a G_CONSTANT (\p LookThroughInstrs == true)
-/// and that constant fits in int64_t, returns its value as well as
-/// the virtual register defined by this G_CONSTANT.
-/// When \p LookThroughInstrs == false, this function behaves like
+/// instructions rooted on a G_F/CONSTANT (\p LookThroughInstrs == true)
+/// and that constant fits in int64_t, returns its value as well as the
+/// virtual register defined by this G_F/CONSTANT.
+/// When \p LookThroughInstrs == false this function behaves like
/// getConstantVRegVal.
+/// When \p HandleFConstants == false the function bails on G_FCONSTANTs.
Optional<ValueAndVReg>
getConstantVRegValWithLookThrough(unsigned VReg, const MachineRegisterInfo &MRI,
- bool LookThroughInstrs = true);
+ bool LookThroughInstrs = true,
+ bool HandleFConstants = true);
const ConstantFP* getConstantFPVRegVal(unsigned VReg,
const MachineRegisterInfo &MRI);
@@ -151,6 +155,9 @@ Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const unsigned Op1,
const unsigned Op2,
const MachineRegisterInfo &MRI);
+Optional<APInt> ConstantFoldExtOp(unsigned Opcode, const unsigned Op1,
+ uint64_t Imm, const MachineRegisterInfo &MRI);
+
/// Returns true if \p Val can be assumed to never be a NaN. If \p SNaN is true,
/// this returns if \p Val can be assumed to never be a signaling NaN.
bool isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,
@@ -161,5 +168,10 @@ inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) {
return isKnownNeverNaN(Val, MRI, true);
}
+/// Get a rough equivalent of an MVT for a given LLT.
+MVT getMVTForLLT(LLT Ty);
+/// Get a rough equivalent of an LLT for a given MVT.
+LLT getLLTForMVT(MVT Ty);
+
} // End namespace llvm.
#endif
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-15 09:27:08 +0000