summaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/GlobalISel
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/GlobalISel')
-rw-r--r--llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp7
-rw-r--r--llvm/lib/CodeGen/GlobalISel/CallLowering.cpp6
-rw-r--r--llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp172
-rw-r--r--llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp74
-rw-r--r--llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp70
-rw-r--r--llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp2
-rw-r--r--llvm/lib/CodeGen/GlobalISel/Legalizer.cpp131
-rw-r--r--llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp248
-rw-r--r--llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp16
-rw-r--r--llvm/lib/CodeGen/GlobalISel/Localizer.cpp13
-rw-r--r--llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp19
-rw-r--r--llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp15
-rw-r--r--llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp22
-rw-r--r--llvm/lib/CodeGen/GlobalISel/Utils.cpp17
14 files changed, 631 insertions, 181 deletions
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
index 7d9d812d34bc..e6abfcdb92cb 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp
@@ -10,11 +10,16 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
#define DEBUG_TYPE "cseinfo"
using namespace llvm;
char llvm::GISelCSEAnalysisWrapperPass::ID = 0;
+GISelCSEAnalysisWrapperPass::GISelCSEAnalysisWrapperPass()
+ : MachineFunctionPass(ID) {
+ initializeGISelCSEAnalysisWrapperPassPass(*PassRegistry::getPassRegistry());
+}
INITIALIZE_PASS_BEGIN(GISelCSEAnalysisWrapperPass, DEBUG_TYPE,
"Analysis containing CSE Info", false, true)
INITIALIZE_PASS_END(GISelCSEAnalysisWrapperPass, DEBUG_TYPE,
@@ -52,7 +57,7 @@ bool CSEConfigFull::shouldCSEOpc(unsigned Opc) {
case TargetOpcode::G_ANYEXT:
case TargetOpcode::G_UNMERGE_VALUES:
case TargetOpcode::G_TRUNC:
- case TargetOpcode::G_GEP:
+ case TargetOpcode::G_PTR_ADD:
return true;
}
return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index cdad92f7db4f..4c2dbdd905f3 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -65,7 +65,11 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
Info.SwiftErrorVReg = SwiftErrorVReg;
Info.IsMustTailCall = CS.isMustTailCall();
Info.IsTailCall = CS.isTailCall() &&
- isInTailCallPosition(CS, MIRBuilder.getMF().getTarget());
+ isInTailCallPosition(CS, MIRBuilder.getMF().getTarget()) &&
+ (MIRBuilder.getMF()
+ .getFunction()
+ .getFnAttribute("disable-tail-calls")
+ .getValueAsString() != "true");
Info.IsVarArg = CS.getFunctionType()->isVarArg();
return lowerCall(MIRBuilder, Info);
}
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 854769d283f7..a103e8e4e6e0 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -74,12 +74,35 @@ bool CombinerHelper::matchCombineCopy(MachineInstr &MI) {
return false;
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = MI.getOperand(1).getReg();
+
+ // Give up if either DstReg or SrcReg is a physical register.
+ if (Register::isPhysicalRegister(DstReg) ||
+ Register::isPhysicalRegister(SrcReg))
+ return false;
+
+ // Give up the types don't match.
LLT DstTy = MRI.getType(DstReg);
LLT SrcTy = MRI.getType(SrcReg);
- // Simple Copy Propagation.
- // a(sx) = COPY b(sx) -> Replace all uses of a with b.
- if (DstTy.isValid() && SrcTy.isValid() && DstTy == SrcTy)
+ // Give up if one has a valid LLT, but the other doesn't.
+ if (DstTy.isValid() != SrcTy.isValid())
+ return false;
+ // Give up if the types don't match.
+ if (DstTy.isValid() && SrcTy.isValid() && DstTy != SrcTy)
+ return false;
+
+ // Get the register banks and classes.
+ const RegisterBank *DstBank = MRI.getRegBankOrNull(DstReg);
+ const RegisterBank *SrcBank = MRI.getRegBankOrNull(SrcReg);
+ const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg);
+ const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg);
+
+ // Replace if the register constraints match.
+ if ((SrcRC == DstRC) && (SrcBank == DstBank))
+ return true;
+ // Replace if DstReg has no constraints.
+ if (!DstBank && !DstRC)
return true;
+
return false;
}
void CombinerHelper::applyCombineCopy(MachineInstr &MI) {
@@ -109,10 +132,7 @@ bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef,
// Walk over all the operands of concat vectors and check if they are
// build_vector themselves or undef.
// Then collect their operands in Ops.
- for (const MachineOperand &MO : MI.operands()) {
- // Skip the instruction definition.
- if (MO.isDef())
- continue;
+ for (const MachineOperand &MO : MI.uses()) {
Register Reg = MO.getReg();
MachineInstr *Def = MRI.getVRegDef(Reg);
assert(Def && "Operand not defined");
@@ -121,12 +141,8 @@ bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef,
IsUndef = false;
// Remember the operands of the build_vector to fold
// them into the yet-to-build flattened concat vectors.
- for (const MachineOperand &BuildVecMO : Def->operands()) {
- // Skip the definition.
- if (BuildVecMO.isDef())
- continue;
+ for (const MachineOperand &BuildVecMO : Def->uses())
Ops.push_back(BuildVecMO.getReg());
- }
break;
case TargetOpcode::G_IMPLICIT_DEF: {
LLT OpType = MRI.getType(Reg);
@@ -189,8 +205,11 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI,
LLT DstType = MRI.getType(MI.getOperand(0).getReg());
Register Src1 = MI.getOperand(1).getReg();
LLT SrcType = MRI.getType(Src1);
- unsigned DstNumElts = DstType.getNumElements();
- unsigned SrcNumElts = SrcType.getNumElements();
+ // As bizarre as it may look, shuffle vector can actually produce
+ // scalar! This is because at the IR level a <1 x ty> shuffle
+ // vector is perfectly valid.
+ unsigned DstNumElts = DstType.isVector() ? DstType.getNumElements() : 1;
+ unsigned SrcNumElts = SrcType.isVector() ? SrcType.getNumElements() : 1;
// If the resulting vector is smaller than the size of the source
// vectors being concatenated, we won't be able to replace the
@@ -199,7 +218,15 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI,
// Note: We may still be able to produce a concat_vectors fed by
// extract_vector_elt and so on. It is less clear that would
// be better though, so don't bother for now.
- if (DstNumElts < 2 * SrcNumElts)
+ //
+ // If the destination is a scalar, the size of the sources doesn't
+ // matter. we will lower the shuffle to a plain copy. This will
+ // work only if the source and destination have the same size. But
+ // that's covered by the next condition.
+ //
+ // TODO: If the size between the source and destination don't match
+ // we could still emit an extract vector element in that case.
+ if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1)
return false;
// Check that the shuffle mask can be broken evenly between the
@@ -212,8 +239,7 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI,
// vectors.
unsigned NumConcat = DstNumElts / SrcNumElts;
SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(MI.getOperand(3).getShuffleMask(), Mask);
+ ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
for (unsigned i = 0; i != DstNumElts; ++i) {
int Idx = Mask[i];
// Undef value.
@@ -254,7 +280,10 @@ void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
Builder.setInsertPt(*MI.getParent(), MI);
Register NewDstReg = MRI.cloneVirtualRegister(DstReg);
- Builder.buildConcatVectors(NewDstReg, Ops);
+ if (Ops.size() == 1)
+ Builder.buildCopy(NewDstReg, Ops[0]);
+ else
+ Builder.buildMerge(NewDstReg, Ops);
MI.eraseFromParent();
replaceRegWith(MRI, DstReg, NewDstReg);
@@ -571,7 +600,7 @@ bool CombinerHelper::findPostIndexCandidate(MachineInstr &MI, Register &Addr,
LLVM_DEBUG(dbgs() << "Searching for post-indexing opportunity for: " << MI);
for (auto &Use : MRI.use_instructions(Base)) {
- if (Use.getOpcode() != TargetOpcode::G_GEP)
+ if (Use.getOpcode() != TargetOpcode::G_PTR_ADD)
continue;
Offset = Use.getOperand(2).getReg();
@@ -597,8 +626,8 @@ bool CombinerHelper::findPostIndexCandidate(MachineInstr &MI, Register &Addr,
// forming an indexed one.
bool MemOpDominatesAddrUses = true;
- for (auto &GEPUse : MRI.use_instructions(Use.getOperand(0).getReg())) {
- if (!dominates(MI, GEPUse)) {
+ for (auto &PtrAddUse : MRI.use_instructions(Use.getOperand(0).getReg())) {
+ if (!dominates(MI, PtrAddUse)) {
MemOpDominatesAddrUses = false;
break;
}
@@ -631,7 +660,7 @@ bool CombinerHelper::findPreIndexCandidate(MachineInstr &MI, Register &Addr,
#endif
Addr = MI.getOperand(1).getReg();
- MachineInstr *AddrDef = getOpcodeDef(TargetOpcode::G_GEP, Addr, MRI);
+ MachineInstr *AddrDef = getOpcodeDef(TargetOpcode::G_PTR_ADD, Addr, MRI);
if (!AddrDef || MRI.hasOneUse(Addr))
return false;
@@ -667,8 +696,8 @@ bool CombinerHelper::findPreIndexCandidate(MachineInstr &MI, Register &Addr,
}
}
- // FIXME: check whether all uses of the base pointer are constant GEPs. That
- // might allow us to end base's liveness here by adjusting the constant.
+ // FIXME: check whether all uses of the base pointer are constant PtrAdds.
+ // That might allow us to end base's liveness here by adjusting the constant.
for (auto &UseMI : MRI.use_instructions(Addr)) {
if (!dominates(MI, UseMI)) {
@@ -681,18 +710,36 @@ bool CombinerHelper::findPreIndexCandidate(MachineInstr &MI, Register &Addr,
}
bool CombinerHelper::tryCombineIndexedLoadStore(MachineInstr &MI) {
+ IndexedLoadStoreMatchInfo MatchInfo;
+ if (matchCombineIndexedLoadStore(MI, MatchInfo)) {
+ applyCombineIndexedLoadStore(MI, MatchInfo);
+ return true;
+ }
+ return false;
+}
+
+bool CombinerHelper::matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) {
unsigned Opcode = MI.getOpcode();
if (Opcode != TargetOpcode::G_LOAD && Opcode != TargetOpcode::G_SEXTLOAD &&
Opcode != TargetOpcode::G_ZEXTLOAD && Opcode != TargetOpcode::G_STORE)
return false;
- bool IsStore = Opcode == TargetOpcode::G_STORE;
- Register Addr, Base, Offset;
- bool IsPre = findPreIndexCandidate(MI, Addr, Base, Offset);
- if (!IsPre && !findPostIndexCandidate(MI, Addr, Base, Offset))
+ MatchInfo.IsPre = findPreIndexCandidate(MI, MatchInfo.Addr, MatchInfo.Base,
+ MatchInfo.Offset);
+ if (!MatchInfo.IsPre &&
+ !findPostIndexCandidate(MI, MatchInfo.Addr, MatchInfo.Base,
+ MatchInfo.Offset))
return false;
+ return true;
+}
+void CombinerHelper::applyCombineIndexedLoadStore(
+ MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) {
+ MachineInstr &AddrDef = *MRI.getUniqueVRegDef(MatchInfo.Addr);
+ MachineIRBuilder MIRBuilder(MI);
+ unsigned Opcode = MI.getOpcode();
+ bool IsStore = Opcode == TargetOpcode::G_STORE;
unsigned NewOpcode;
switch (Opcode) {
case TargetOpcode::G_LOAD:
@@ -711,25 +758,22 @@ bool CombinerHelper::tryCombineIndexedLoadStore(MachineInstr &MI) {
llvm_unreachable("Unknown load/store opcode");
}
- MachineInstr &AddrDef = *MRI.getUniqueVRegDef(Addr);
- MachineIRBuilder MIRBuilder(MI);
auto MIB = MIRBuilder.buildInstr(NewOpcode);
if (IsStore) {
- MIB.addDef(Addr);
+ MIB.addDef(MatchInfo.Addr);
MIB.addUse(MI.getOperand(0).getReg());
} else {
MIB.addDef(MI.getOperand(0).getReg());
- MIB.addDef(Addr);
+ MIB.addDef(MatchInfo.Addr);
}
- MIB.addUse(Base);
- MIB.addUse(Offset);
- MIB.addImm(IsPre);
+ MIB.addUse(MatchInfo.Base);
+ MIB.addUse(MatchInfo.Offset);
+ MIB.addImm(MatchInfo.IsPre);
MI.eraseFromParent();
AddrDef.eraseFromParent();
LLVM_DEBUG(dbgs() << " Combinined to indexed operation");
- return true;
}
bool CombinerHelper::matchElideBrByInvertingCond(MachineInstr &MI) {
@@ -1016,7 +1060,7 @@ bool CombinerHelper::optimizeMemset(MachineInstr &MI, Register Dst, Register Val
if (DstOff != 0) {
auto Offset =
MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), DstOff);
- Ptr = MIB.buildGEP(PtrTy, Dst, Offset).getReg(0);
+ Ptr = MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0);
}
MIB.buildStore(Value, Ptr, *StoreMMO);
@@ -1121,13 +1165,13 @@ bool CombinerHelper::optimizeMemcpy(MachineInstr &MI, Register Dst,
if (CurrOffset != 0) {
Offset = MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), CurrOffset)
.getReg(0);
- LoadPtr = MIB.buildGEP(PtrTy, Src, Offset).getReg(0);
+ LoadPtr = MIB.buildPtrAdd(PtrTy, Src, Offset).getReg(0);
}
auto LdVal = MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO);
// Create the store.
Register StorePtr =
- CurrOffset == 0 ? Dst : MIB.buildGEP(PtrTy, Dst, Offset).getReg(0);
+ CurrOffset == 0 ? Dst : MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0);
MIB.buildStore(LdVal, StorePtr, *StoreMMO);
CurrOffset += CopyTy.getSizeInBytes();
Size -= CopyTy.getSizeInBytes();
@@ -1218,7 +1262,7 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst,
if (CurrOffset != 0) {
auto Offset =
MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), CurrOffset);
- LoadPtr = MIB.buildGEP(PtrTy, Src, Offset).getReg(0);
+ LoadPtr = MIB.buildPtrAdd(PtrTy, Src, Offset).getReg(0);
}
LoadVals.push_back(MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO).getReg(0));
CurrOffset += CopyTy.getSizeInBytes();
@@ -1235,7 +1279,7 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst,
if (CurrOffset != 0) {
auto Offset =
MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), CurrOffset);
- StorePtr = MIB.buildGEP(PtrTy, Dst, Offset).getReg(0);
+ StorePtr = MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0);
}
MIB.buildStore(LoadVals[I], StorePtr, *StoreMMO);
CurrOffset += CopyTy.getSizeInBytes();
@@ -1295,6 +1339,52 @@ bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
return false;
}
+bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
+ PtrAddChain &MatchInfo) {
+ // We're trying to match the following pattern:
+ // %t1 = G_PTR_ADD %base, G_CONSTANT imm1
+ // %root = G_PTR_ADD %t1, G_CONSTANT imm2
+ // -->
+ // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2)
+
+ if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)
+ return false;
+
+ Register Add2 = MI.getOperand(1).getReg();
+ Register Imm1 = MI.getOperand(2).getReg();
+ auto MaybeImmVal = getConstantVRegValWithLookThrough(Imm1, MRI);
+ if (!MaybeImmVal)
+ return false;
+
+ MachineInstr *Add2Def = MRI.getUniqueVRegDef(Add2);
+ if (!Add2Def || Add2Def->getOpcode() != TargetOpcode::G_PTR_ADD)
+ return false;
+
+ Register Base = Add2Def->getOperand(1).getReg();
+ Register Imm2 = Add2Def->getOperand(2).getReg();
+ auto MaybeImm2Val = getConstantVRegValWithLookThrough(Imm2, MRI);
+ if (!MaybeImm2Val)
+ return false;
+
+ // Pass the combined immediate to the apply function.
+ MatchInfo.Imm = MaybeImmVal->Value + MaybeImm2Val->Value;
+ MatchInfo.Base = Base;
+ return true;
+}
+
+bool CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
+ PtrAddChain &MatchInfo) {
+ assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
+ MachineIRBuilder MIB(MI);
+ LLT OffsetTy = MRI.getType(MI.getOperand(2).getReg());
+ auto NewOffset = MIB.buildConstant(OffsetTy, MatchInfo.Imm);
+ Observer.changingInstr(MI);
+ MI.getOperand(1).setReg(MatchInfo.Base);
+ MI.getOperand(2).setReg(NewOffset.getReg(0));
+ Observer.changedInstr(MI);
+ return true;
+}
+
bool CombinerHelper::tryCombine(MachineInstr &MI) {
if (tryCombineCopy(MI))
return true;
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index be8efa8795f3..64023ecfad82 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -179,8 +179,8 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
Known.Zero = KnownZeroOut;
break;
}
- case TargetOpcode::G_GEP: {
- // G_GEP is like G_ADD. FIXME: Is this true for all targets?
+ case TargetOpcode::G_PTR_ADD: {
+ // G_PTR_ADD is like G_ADD. FIXME: Is this true for all targets?
LLT Ty = MRI.getType(MI.getOperand(1).getReg());
if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
break;
@@ -373,6 +373,76 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
<< Known.One.toString(16, false) << "\n");
}
+unsigned GISelKnownBits::computeNumSignBits(Register R,
+ const APInt &DemandedElts,
+ unsigned Depth) {
+ MachineInstr &MI = *MRI.getVRegDef(R);
+ unsigned Opcode = MI.getOpcode();
+
+ if (Opcode == TargetOpcode::G_CONSTANT)
+ return MI.getOperand(1).getCImm()->getValue().getNumSignBits();
+
+ if (Depth == getMaxDepth())
+ return 1;
+
+ if (!DemandedElts)
+ return 1; // No demanded elts, better to assume we don't know anything.
+
+ LLT DstTy = MRI.getType(R);
+
+ // Handle the case where this is called on a register that does not have a
+ // type constraint. This is unlikely to occur except by looking through copies
+ // but it is possible for the initial register being queried to be in this
+ // state.
+ if (!DstTy.isValid())
+ return 1;
+
+ switch (Opcode) {
+ case TargetOpcode::COPY: {
+ MachineOperand &Src = MI.getOperand(1);
+ if (Src.getReg().isVirtual() && Src.getSubReg() == 0 &&
+ MRI.getType(Src.getReg()).isValid()) {
+ // Don't increment Depth for this one since we didn't do any work.
+ return computeNumSignBits(Src.getReg(), DemandedElts, Depth);
+ }
+
+ return 1;
+ }
+ case TargetOpcode::G_SEXT: {
+ Register Src = MI.getOperand(1).getReg();
+ LLT SrcTy = MRI.getType(Src);
+ unsigned Tmp = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits();
+ return computeNumSignBits(Src, DemandedElts, Depth + 1) + Tmp;
+ }
+ case TargetOpcode::G_TRUNC: {
+ Register Src = MI.getOperand(1).getReg();
+ LLT SrcTy = MRI.getType(Src);
+
+ // Check if the sign bits of source go down as far as the truncated value.
+ unsigned DstTyBits = DstTy.getScalarSizeInBits();
+ unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
+ unsigned NumSrcSignBits = computeNumSignBits(Src, DemandedElts, Depth + 1);
+ if (NumSrcSignBits > (NumSrcBits - DstTyBits))
+ return NumSrcSignBits - (NumSrcBits - DstTyBits);
+ break;
+ }
+ default:
+ break;
+ }
+
+ // TODO: Handle target instructions
+ // TODO: Fall back to known bits
+ return 1;
+}
+
+unsigned GISelKnownBits::computeNumSignBits(Register R, unsigned Depth) {
+ LLT Ty = MRI.getType(R);
+ APInt DemandedElts = Ty.isVector()
+ ? APInt::getAllOnesValue(Ty.getNumElements())
+ : APInt(1, 1);
+ return computeNumSignBits(R, DemandedElts, Depth);
+}
+
void GISelKnownBitsAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 45cef4aca888..17eca2b0301c 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -56,6 +56,7 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
#include "llvm/MC/MCContext.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
@@ -224,12 +225,12 @@ int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
if (FrameIndices.find(&AI) != FrameIndices.end())
return FrameIndices[&AI];
- unsigned ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
- unsigned Size =
+ uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
+ uint64_t Size =
ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
// Always allocate at least one byte.
- Size = std::max(Size, 1u);
+ Size = std::max<uint64_t>(Size, 1u);
unsigned Alignment = AI.getAlignment();
if (!Alignment)
@@ -466,7 +467,7 @@ bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
return true;
}
- SL->findJumpTables(Clusters, &SI, DefaultMBB);
+ SL->findJumpTables(Clusters, &SI, DefaultMBB, nullptr, nullptr);
LLVM_DEBUG({
dbgs() << "Case clusters: ";
@@ -885,13 +886,15 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
for (unsigned i = 0; i < Regs.size(); ++i) {
Register Addr;
- MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
+ MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
unsigned BaseAlign = getMemOpAlignment(LI);
+ AAMDNodes AAMetadata;
+ LI.getAAMetadata(AAMetadata);
auto MMO = MF->getMachineMemOperand(
Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
- MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), Ranges,
+ MinAlign(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges,
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -926,13 +929,15 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
for (unsigned i = 0; i < Vals.size(); ++i) {
Register Addr;
- MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
+ MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
unsigned BaseAlign = getMemOpAlignment(SI);
+ AAMDNodes AAMetadata;
+ SI.getAAMetadata(AAMetadata);
auto MMO = MF->getMachineMemOperand(
Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
- MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
+ MinAlign(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr,
SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
}
@@ -1080,8 +1085,8 @@ bool IRTranslator::translateGetElementPtr(const User &U,
if (Offset != 0) {
LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
- BaseReg =
- MIRBuilder.buildGEP(PtrTy, BaseReg, OffsetMIB.getReg(0)).getReg(0);
+ BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
+ .getReg(0);
Offset = 0;
}
@@ -1100,14 +1105,14 @@ bool IRTranslator::translateGetElementPtr(const User &U,
} else
GepOffsetReg = IdxReg;
- BaseReg = MIRBuilder.buildGEP(PtrTy, BaseReg, GepOffsetReg).getReg(0);
+ BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
}
}
if (Offset != 0) {
auto OffsetMIB =
MIRBuilder.buildConstant(getLLTForType(*OffsetIRTy, *DL), Offset);
- MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
+ MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
return true;
}
@@ -1251,6 +1256,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return TargetOpcode::G_FSQRT;
case Intrinsic::trunc:
return TargetOpcode::G_INTRINSIC_TRUNC;
+ case Intrinsic::readcyclecounter:
+ return TargetOpcode::G_READCYCLECOUNTER;
}
return Intrinsic::not_intrinsic;
}
@@ -1412,7 +1419,8 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
- TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) {
+ TLI.isFMAFasterThanFMulAndFAdd(*MF,
+ TLI.getValueType(*DL, CI.getType()))) {
// TODO: Revisit this to see if we should move this part of the
// lowering to the combiner.
MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2},
@@ -1518,6 +1526,13 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
case Intrinsic::sideeffect:
// Discard annotate attributes, assumptions, and artificial side-effects.
return true;
+ case Intrinsic::read_register: {
+ Value *Arg = CI.getArgOperand(0);
+ MIRBuilder.buildInstr(TargetOpcode::G_READ_REGISTER)
+ .addDef(getOrCreateVReg(CI))
+ .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
+ return true;
+ }
}
return false;
}
@@ -1587,7 +1602,13 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
const Function *F = CI.getCalledFunction();
// FIXME: support Windows dllimport function calls.
- if (F && F->hasDLLImportStorageClass())
+ if (F && (F->hasDLLImportStorageClass() ||
+ (MF->getTarget().getTargetTriple().isOSWindows() &&
+ F->hasExternalWeakLinkage())))
+ return false;
+
+ // FIXME: support control flow guard targets.
+ if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
return false;
if (CI.isInlineAsm())
@@ -1683,6 +1704,10 @@ bool IRTranslator::translateInvoke(const User &U,
if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
return false;
+ // FIXME: support control flow guard targets.
+ if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
+ return false;
+
// FIXME: support Windows exception handling.
if (!isa<LandingPadInst>(EHPadBB->front()))
return false;
@@ -1908,11 +1933,14 @@ bool IRTranslator::translateExtractElement(const User &U,
bool IRTranslator::translateShuffleVector(const User &U,
MachineIRBuilder &MIRBuilder) {
+ SmallVector<int, 8> Mask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(U.getOperand(2)), Mask);
+ ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
.addDef(getOrCreateVReg(U))
.addUse(getOrCreateVReg(*U.getOperand(0)))
.addUse(getOrCreateVReg(*U.getOperand(1)))
- .addShuffleMask(cast<Constant>(U.getOperand(2)));
+ .addShuffleMask(MaskAlloc);
return true;
}
@@ -1950,11 +1978,14 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
Register Cmp = getOrCreateVReg(*I.getCompareOperand());
Register NewVal = getOrCreateVReg(*I.getNewValOperand());
+ AAMDNodes AAMetadata;
+ I.getAAMetadata(AAMetadata);
+
MIRBuilder.buildAtomicCmpXchgWithSuccess(
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, DL->getTypeStoreSize(ValType),
- getMemOpAlignment(I), AAMDNodes(), nullptr,
+ getMemOpAlignment(I), AAMetadata, nullptr,
I.getSyncScopeID(), I.getSuccessOrdering(),
I.getFailureOrdering()));
return true;
@@ -2019,12 +2050,15 @@ bool IRTranslator::translateAtomicRMW(const User &U,
break;
}
+ AAMDNodes AAMetadata;
+ I.getAAMetadata(AAMetadata);
+
MIRBuilder.buildAtomicRMW(
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, DL->getTypeStoreSize(ResType),
- getMemOpAlignment(I), AAMDNodes(), nullptr,
- I.getSyncScopeID(), I.getOrdering()));
+ getMemOpAlignment(I), AAMetadata,
+ nullptr, I.getSyncScopeID(), I.getOrdering()));
return true;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp
index 28143b30d4e8..b9c90e69ddb2 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp
@@ -60,7 +60,7 @@ bool InstructionSelector::isBaseWithConstantOffset(
return false;
MachineInstr *RootI = MRI.getVRegDef(Root.getReg());
- if (RootI->getOpcode() != TargetOpcode::G_GEP)
+ if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
return false;
MachineOperand &RHS = RootI->getOperand(2);
diff --git a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
index 1593e21fe07e..e789e4a333dc 100644
--- a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -26,6 +26,7 @@
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"
#include "llvm/Target/TargetMachine.h"
@@ -139,22 +140,13 @@ public:
};
} // namespace
-bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
- // If the ISel pipeline failed, do not bother running that pass.
- if (MF.getProperties().hasProperty(
- MachineFunctionProperties::Property::FailedISel))
- return false;
- LLVM_DEBUG(dbgs() << "Legalize Machine IR for: " << MF.getName() << '\n');
- init(MF);
- const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
- GISelCSEAnalysisWrapper &Wrapper =
- getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
- MachineOptimizationRemarkEmitter MORE(MF, /*MBFI=*/nullptr);
-
- const size_t NumBlocks = MF.size();
+Legalizer::MFResult
+Legalizer::legalizeMachineFunction(MachineFunction &MF, const LegalizerInfo &LI,
+ ArrayRef<GISelChangeObserver *> AuxObservers,
+ MachineIRBuilder &MIRBuilder) {
MachineRegisterInfo &MRI = MF.getRegInfo();
- // Populate Insts
+ // Populate worklists.
InstListTy InstList;
ArtifactListTy ArtifactList;
ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
@@ -177,48 +169,33 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
}
ArtifactList.finalize();
InstList.finalize();
- std::unique_ptr<MachineIRBuilder> MIRBuilder;
- GISelCSEInfo *CSEInfo = nullptr;
- bool EnableCSE = EnableCSEInLegalizer.getNumOccurrences()
- ? EnableCSEInLegalizer
- : TPC.isGISelCSEEnabled();
- if (EnableCSE) {
- MIRBuilder = std::make_unique<CSEMIRBuilder>();
- CSEInfo = &Wrapper.get(TPC.getCSEConfig());
- MIRBuilder->setCSEInfo(CSEInfo);
- } else
- MIRBuilder = std::make_unique<MachineIRBuilder>();
- // This observer keeps the worklist updated.
+ // This observer keeps the worklists updated.
LegalizerWorkListManager WorkListObserver(InstList, ArtifactList);
- // We want both WorkListObserver as well as CSEInfo to observe all changes.
- // Use the wrapper observer.
+ // We want both WorkListObserver as well as all the auxiliary observers (e.g.
+ // CSEInfo) to observe all changes. Use the wrapper observer.
GISelObserverWrapper WrapperObserver(&WorkListObserver);
- if (EnableCSE && CSEInfo)
- WrapperObserver.addObserver(CSEInfo);
+ for (GISelChangeObserver *Observer : AuxObservers)
+ WrapperObserver.addObserver(Observer);
+
// Now install the observer as the delegate to MF.
// This will keep all the observers notified about new insertions/deletions.
RAIIDelegateInstaller DelInstall(MF, &WrapperObserver);
- LegalizerHelper Helper(MF, WrapperObserver, *MIRBuilder.get());
- const LegalizerInfo &LInfo(Helper.getLegalizerInfo());
- LegalizationArtifactCombiner ArtCombiner(*MIRBuilder.get(), MF.getRegInfo(),
- LInfo);
+ LegalizerHelper Helper(MF, LI, WrapperObserver, MIRBuilder);
+ LegalizationArtifactCombiner ArtCombiner(MIRBuilder, MRI, LI);
auto RemoveDeadInstFromLists = [&WrapperObserver](MachineInstr *DeadMI) {
WrapperObserver.erasingInstr(*DeadMI);
};
- auto stopLegalizing = [&](MachineInstr &MI) {
- Helper.MIRBuilder.stopObservingChanges();
- reportGISelFailure(MF, TPC, MORE, "gisel-legalize",
- "unable to legalize instruction", MI);
- };
bool Changed = false;
SmallVector<MachineInstr *, 128> RetryList;
do {
+ LLVM_DEBUG(dbgs() << "=== New Iteration ===\n");
assert(RetryList.empty() && "Expected no instructions in RetryList");
unsigned NumArtifacts = ArtifactList.size();
while (!InstList.empty()) {
MachineInstr &MI = *InstList.pop_back_val();
- assert(isPreISelGenericOpcode(MI.getOpcode()) && "Expecting generic opcode");
+ assert(isPreISelGenericOpcode(MI.getOpcode()) &&
+ "Expecting generic opcode");
if (isTriviallyDead(MI, MRI)) {
LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");
MI.eraseFromParentAndMarkDBGValuesForRemoval();
@@ -234,11 +211,17 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
// legalizing InstList may generate artifacts that allow
// ArtifactCombiner to combine away them.
if (isArtifact(MI)) {
+ LLVM_DEBUG(dbgs() << ".. Not legalized, moving to artifacts retry\n");
+ assert(NumArtifacts == 0 &&
+ "Artifacts are only expected in instruction list starting the "
+ "second iteration, but each iteration starting second must "
+ "start with an empty artifacts list");
+ (void)NumArtifacts;
RetryList.push_back(&MI);
continue;
}
- stopLegalizing(MI);
- return false;
+ Helper.MIRBuilder.stopObservingChanges();
+ return {Changed, &MI};
}
WorkListObserver.printNewInstrs();
Changed |= Res == LegalizerHelper::Legalized;
@@ -246,18 +229,19 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
// Try to combine the instructions in RetryList again if there
// are new artifacts. If not, stop legalizing.
if (!RetryList.empty()) {
- if (ArtifactList.size() > NumArtifacts) {
+ if (!ArtifactList.empty()) {
while (!RetryList.empty())
ArtifactList.insert(RetryList.pop_back_val());
} else {
- MachineInstr *MI = *RetryList.begin();
- stopLegalizing(*MI);
- return false;
+ LLVM_DEBUG(dbgs() << "No new artifacts created, not retrying!\n");
+ Helper.MIRBuilder.stopObservingChanges();
+ return {Changed, RetryList.front()};
}
}
while (!ArtifactList.empty()) {
MachineInstr &MI = *ArtifactList.pop_back_val();
- assert(isPreISelGenericOpcode(MI.getOpcode()) && "Expecting generic opcode");
+ assert(isPreISelGenericOpcode(MI.getOpcode()) &&
+ "Expecting generic opcode");
if (isTriviallyDead(MI, MRI)) {
LLVM_DEBUG(dbgs() << MI << "Is dead\n");
RemoveDeadInstFromLists(&MI);
@@ -265,6 +249,7 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
continue;
}
SmallVector<MachineInstr *, 4> DeadInstructions;
+ LLVM_DEBUG(dbgs() << "Trying to combine: " << MI);
if (ArtCombiner.tryCombineInstruction(MI, DeadInstructions,
WrapperObserver)) {
WorkListObserver.printNewInstrs();
@@ -279,13 +264,58 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
// If this was not an artifact (that could be combined away), this might
// need special handling. Add it to InstList, so when it's processed
// there, it has to be legal or specially handled.
- else
+ else {
+ LLVM_DEBUG(dbgs() << ".. Not combined, moving to instructions list\n");
InstList.insert(&MI);
+ }
}
} while (!InstList.empty());
+ return {Changed, /*FailedOn*/ nullptr};
+}
+
+bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
+ // If the ISel pipeline failed, do not bother running that pass.
+ if (MF.getProperties().hasProperty(
+ MachineFunctionProperties::Property::FailedISel))
+ return false;
+ LLVM_DEBUG(dbgs() << "Legalize Machine IR for: " << MF.getName() << '\n');
+ init(MF);
+ const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
+ GISelCSEAnalysisWrapper &Wrapper =
+ getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
+ MachineOptimizationRemarkEmitter MORE(MF, /*MBFI=*/nullptr);
+
+ const size_t NumBlocks = MF.size();
+
+ std::unique_ptr<MachineIRBuilder> MIRBuilder;
+ GISelCSEInfo *CSEInfo = nullptr;
+ bool EnableCSE = EnableCSEInLegalizer.getNumOccurrences()
+ ? EnableCSEInLegalizer
+ : TPC.isGISelCSEEnabled();
+ if (EnableCSE) {
+ MIRBuilder = std::make_unique<CSEMIRBuilder>();
+ CSEInfo = &Wrapper.get(TPC.getCSEConfig());
+ MIRBuilder->setCSEInfo(CSEInfo);
+ } else
+ MIRBuilder = std::make_unique<MachineIRBuilder>();
+
+ SmallVector<GISelChangeObserver *, 1> AuxObservers;
+ if (EnableCSE && CSEInfo) {
+ // We want CSEInfo in addition to WorkListObserver to observe all changes.
+ AuxObservers.push_back(CSEInfo);
+ }
+
+ const LegalizerInfo &LI = *MF.getSubtarget().getLegalizerInfo();
+ MFResult Result = legalizeMachineFunction(MF, LI, AuxObservers, *MIRBuilder);
+
+ if (Result.FailedOn) {
+ reportGISelFailure(MF, TPC, MORE, "gisel-legalize",
+ "unable to legalize instruction", *Result.FailedOn);
+ return false;
+ }
// For now don't support if new blocks are inserted - we would need to fix the
- // outerloop for that.
+ // outer loop for that.
if (MF.size() != NumBlocks) {
MachineOptimizationRemarkMissed R("gisel-legalize", "GISelFailure",
MF.getFunction().getSubprogram(),
@@ -294,6 +324,5 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
reportGISelFailure(MF, TPC, MORE, R);
return false;
}
-
- return Changed;
+ return Result.Changed;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 21512e543878..667e1a04dc34 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1075,6 +1075,28 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
MI.eraseFromParent();
return Legalized;
}
+ case TargetOpcode::G_BSWAP:
+ case TargetOpcode::G_BITREVERSE: {
+ if (SizeOp0 % NarrowSize != 0)
+ return UnableToLegalize;
+
+ Observer.changingInstr(MI);
+ SmallVector<Register, 2> SrcRegs, DstRegs;
+ unsigned NumParts = SizeOp0 / NarrowSize;
+ extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
+
+ for (unsigned i = 0; i < NumParts; ++i) {
+ auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
+ {SrcRegs[NumParts - 1 - i]});
+ DstRegs.push_back(DstPart.getReg(0));
+ }
+
+ MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
+
+ Observer.changedInstr(MI);
+ MI.eraseFromParent();
+ return Legalized;
+ }
}
}
@@ -1675,7 +1697,15 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
case TargetOpcode::G_CONSTANT: {
MachineOperand &SrcMO = MI.getOperand(1);
LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
- const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
+ unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
+ MRI.getType(MI.getOperand(0).getReg()));
+ assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
+ ExtOpc == TargetOpcode::G_ANYEXT) &&
+ "Illegal Extend");
+ const APInt &SrcVal = SrcMO.getCImm()->getValue();
+ const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
+ ? SrcVal.sext(WideTy.getSizeInBits())
+ : SrcVal.zext(WideTy.getSizeInBits());
Observer.changingInstr(MI);
SrcMO.setCImm(ConstantInt::get(Ctx, Val));
@@ -1748,8 +1778,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changedInstr(MI);
return Legalized;
- case TargetOpcode::G_GEP:
- assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
+ case TargetOpcode::G_PTR_ADD:
+ assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
Observer.changingInstr(MI);
widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Observer.changedInstr(MI);
@@ -1789,10 +1819,35 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx != 2)
return UnableToLegalize;
Observer.changingInstr(MI);
+ // TODO: Probably should be zext
widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Observer.changedInstr(MI);
return Legalized;
}
+ case TargetOpcode::G_INSERT_VECTOR_ELT: {
+ if (TypeIdx == 1) {
+ Observer.changingInstr(MI);
+
+ Register VecReg = MI.getOperand(1).getReg();
+ LLT VecTy = MRI.getType(VecReg);
+ LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy);
+
+ widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
+ widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
+ widenScalarDst(MI, WideVecTy, 0);
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
+
+ if (TypeIdx == 2) {
+ Observer.changingInstr(MI);
+ // TODO: Probably should be zext
+ widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
+ Observer.changedInstr(MI);
+ }
+
+ return Legalized;
+ }
case TargetOpcode::G_FADD:
case TargetOpcode::G_FMUL:
case TargetOpcode::G_FSUB:
@@ -1998,6 +2053,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
}
case TargetOpcode::G_FMAD:
return lowerFMad(MI);
+ case TargetOpcode::G_INTRINSIC_ROUND:
+ return lowerIntrinsicRound(MI);
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
Register OldValRes = MI.getOperand(0).getReg();
Register SuccessRes = MI.getOperand(1).getReg();
@@ -2058,8 +2115,9 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
auto OffsetCst =
MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
- Register GEPReg = MRI.createGenericVirtualRegister(PtrTy);
- auto SmallPtr = MIRBuilder.buildGEP(GEPReg, PtrReg, OffsetCst.getReg(0));
+ Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
+ auto SmallPtr =
+ MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
*SmallMMO);
@@ -2083,7 +2141,7 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
default:
llvm_unreachable("Unexpected opcode");
case TargetOpcode::G_LOAD:
- MIRBuilder.buildAnyExt(DstReg, TmpReg);
+ MIRBuilder.buildExtOrTrunc(TargetOpcode::G_ANYEXT, DstReg, TmpReg);
break;
case TargetOpcode::G_SEXTLOAD:
MIRBuilder.buildSExt(DstReg, TmpReg);
@@ -2126,12 +2184,13 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
- // Generate the GEP and truncating stores.
+ // Generate the PtrAdd and truncating stores.
LLT PtrTy = MRI.getType(PtrReg);
auto OffsetCst =
MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
- Register GEPReg = MRI.createGenericVirtualRegister(PtrTy);
- auto SmallPtr = MIRBuilder.buildGEP(GEPReg, PtrReg, OffsetCst.getReg(0));
+ Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
+ auto SmallPtr =
+ MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
MachineFunction &MF = MIRBuilder.getMF();
MachineMemOperand *LargeMMO =
@@ -2254,6 +2313,12 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
return lowerExtract(MI);
case G_INSERT:
return lowerInsert(MI);
+ case G_BSWAP:
+ return lowerBswap(MI);
+ case G_BITREVERSE:
+ return lowerBitreverse(MI);
+ case G_READ_REGISTER:
+ return lowerReadRegister(MI);
}
}
@@ -2883,7 +2948,7 @@ LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
unsigned ByteOffset = Offset / 8;
Register NewAddrReg;
- MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
+ MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
MachineMemOperand *NewMMO =
MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
@@ -2960,6 +3025,9 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
case G_BSWAP:
case G_BITREVERSE:
case G_SDIV:
+ case G_UDIV:
+ case G_SREM:
+ case G_UREM:
case G_SMIN:
case G_SMAX:
case G_UMIN:
@@ -3259,7 +3327,13 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
case TargetOpcode::G_SMIN:
case TargetOpcode::G_SMAX:
case TargetOpcode::G_UMIN:
- case TargetOpcode::G_UMAX: {
+ case TargetOpcode::G_UMAX:
+ case TargetOpcode::G_FMINNUM:
+ case TargetOpcode::G_FMAXNUM:
+ case TargetOpcode::G_FMINNUM_IEEE:
+ case TargetOpcode::G_FMAXNUM_IEEE:
+ case TargetOpcode::G_FMINIMUM:
+ case TargetOpcode::G_FMAXIMUM: {
Observer.changingInstr(MI);
moreElementsVectorSrc(MI, MoreTy, 1);
moreElementsVectorSrc(MI, MoreTy, 2);
@@ -3352,7 +3426,7 @@ void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
Factors.push_back(Umulh.getReg(0));
}
- // Add CarrySum from additons calculated for previous DstIdx.
+ // Add CarrySum from additions calculated for previous DstIdx.
if (DstIdx != 1) {
Factors.push_back(CarrySumPrevDstIdx);
}
@@ -3824,6 +3898,14 @@ LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
LLT DstTy = MRI.getType(Dst);
LLT SrcTy = MRI.getType(Src);
+ if (SrcTy == LLT::scalar(1)) {
+ auto True = MIRBuilder.buildFConstant(DstTy, 1.0);
+ auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
+ MIRBuilder.buildSelect(Dst, Src, True, False);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+
if (SrcTy != LLT::scalar(64))
return UnableToLegalize;
@@ -3849,6 +3931,14 @@ LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
const LLT S32 = LLT::scalar(32);
const LLT S1 = LLT::scalar(1);
+ if (SrcTy == S1) {
+ auto True = MIRBuilder.buildFConstant(DstTy, -1.0);
+ auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
+ MIRBuilder.buildSelect(Dst, Src, True, False);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+
if (SrcTy != S64)
return UnableToLegalize;
@@ -3910,8 +4000,10 @@ LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
+ const LLT S1 = LLT::scalar(1);
+
MachineInstrBuilder FCMP =
- MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, DstTy, Src, Threshold);
+ MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold);
MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
MI.eraseFromParent();
@@ -4042,6 +4134,33 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
}
LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
+ Register DstReg = MI.getOperand(0).getReg();
+ Register SrcReg = MI.getOperand(1).getReg();
+ unsigned Flags = MI.getFlags();
+ LLT Ty = MRI.getType(DstReg);
+ const LLT CondTy = Ty.changeElementSize(1);
+
+ // result = trunc(src);
+ // if (src < 0.0 && src != result)
+ // result += -1.0.
+
+ auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
+ auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags);
+
+ auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy,
+ SrcReg, Zero, Flags);
+ auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy,
+ SrcReg, Trunc, Flags);
+ auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc);
+ auto AddVal = MIRBuilder.buildSITOFP(Ty, And);
+
+ MIRBuilder.buildFAdd(DstReg, Trunc, AddVal);
+ MI.eraseFromParent();
+ return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
const unsigned NumDst = MI.getNumOperands() - 1;
const Register SrcReg = MI.getOperand(NumDst).getReg();
@@ -4083,10 +4202,7 @@ LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
LLT DstTy = MRI.getType(DstReg);
LLT IdxTy = LLT::scalar(32);
- const Constant *ShufMask = MI.getOperand(3).getShuffleMask();
-
- SmallVector<int, 32> Mask;
- ShuffleVectorInst::getShuffleMask(ShufMask, Mask);
+ ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
if (DstTy.isScalar()) {
if (Src0Ty.isVector())
@@ -4151,7 +4267,7 @@ LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
// Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
// have to generate an extra instruction to negate the alloc and then use
- // G_GEP to add the negative offset.
+ // G_PTR_ADD to add the negative offset.
auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
if (Align) {
APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true);
@@ -4275,3 +4391,99 @@ LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
MI.eraseFromParent();
return Legalized;
}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerBswap(MachineInstr &MI) {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ const LLT Ty = MRI.getType(Src);
+ unsigned SizeInBytes = Ty.getSizeInBytes();
+ unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
+
+ // Swap most and least significant byte, set remaining bytes in Res to zero.
+ auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt);
+ auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt);
+ auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
+ auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft);
+
+ // Set i-th high/low byte in Res to i-th low/high byte from Src.
+ for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
+ // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
+ APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
+ auto Mask = MIRBuilder.buildConstant(Ty, APMask);
+ auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i);
+ // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
+ auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask);
+ auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt);
+ Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft);
+ // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
+ auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
+ auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask);
+ Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight);
+ }
+ Res.getInstr()->getOperand(0).setReg(Dst);
+
+ MI.eraseFromParent();
+ return Legalized;
+}
+
+//{ (Src & Mask) >> N } | { (Src << N) & Mask }
+static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
+ MachineInstrBuilder Src, APInt Mask) {
+ const LLT Ty = Dst.getLLTTy(*B.getMRI());
+ MachineInstrBuilder C_N = B.buildConstant(Ty, N);
+ MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask);
+ auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
+ auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0);
+ return B.buildOr(Dst, LHS, RHS);
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ const LLT Ty = MRI.getType(Src);
+ unsigned Size = Ty.getSizeInBits();
+
+ MachineInstrBuilder BSWAP =
+ MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src});
+
+ // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
+ // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
+ // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
+ MachineInstrBuilder Swap4 =
+ SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0)));
+
+ // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
+ // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
+ // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
+ MachineInstrBuilder Swap2 =
+ SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC)));
+
+ // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7
+ // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
+ // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
+ SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA)));
+
+ MI.eraseFromParent();
+ return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerReadRegister(MachineInstr &MI) {
+ Register Dst = MI.getOperand(0).getReg();
+ const LLT Ty = MRI.getType(Dst);
+ const MDString *RegStr = cast<MDString>(
+ cast<MDNode>(MI.getOperand(1).getMetadata())->getOperand(0));
+
+ MachineFunction &MF = MIRBuilder.getMF();
+ const TargetSubtargetInfo &STI = MF.getSubtarget();
+ const TargetLowering *TLI = STI.getTargetLowering();
+ Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty, MF);
+ if (!Reg.isValid())
+ return UnableToLegalize;
+
+ MIRBuilder.buildCopy(Dst, Reg);
+ MI.eraseFromParent();
+ return Legalized;
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 70045512fae5..02f6b39e0905 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -128,24 +128,26 @@ static bool mutationIsSane(const LegalizeRule &Rule,
switch (Rule.getAction()) {
case FewerElements:
- case MoreElements: {
if (!OldTy.isVector())
return false;
-
+ LLVM_FALLTHROUGH;
+ case MoreElements: {
+ // MoreElements can go from scalar to vector.
+ const unsigned OldElts = OldTy.isVector() ? OldTy.getNumElements() : 1;
if (NewTy.isVector()) {
if (Rule.getAction() == FewerElements) {
// Make sure the element count really decreased.
- if (NewTy.getNumElements() >= OldTy.getNumElements())
+ if (NewTy.getNumElements() >= OldElts)
return false;
} else {
// Make sure the element count really increased.
- if (NewTy.getNumElements() <= OldTy.getNumElements())
+ if (NewTy.getNumElements() <= OldElts)
return false;
}
}
// Make sure the element type didn't change.
- return NewTy.getScalarType() == OldTy.getElementType();
+ return NewTy.getScalarType() == OldTy.getScalarType();
}
case NarrowScalar:
case WidenScalar: {
@@ -685,6 +687,10 @@ bool LegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
return true;
}
+unsigned LegalizerInfo::getExtOpcodeForWideningConstant(LLT SmallTy) const {
+ return SmallTy.isByteSized() ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
+}
+
/// \pre Type indices of every opcode form a dense set starting from 0.
void LegalizerInfo::verify(const MCInstrInfo &MII) const {
#ifndef NDEBUG
diff --git a/llvm/lib/CodeGen/GlobalISel/Localizer.cpp b/llvm/lib/CodeGen/GlobalISel/Localizer.cpp
index f882ecbf5db3..1c4a668e5f31 100644
--- a/llvm/lib/CodeGen/GlobalISel/Localizer.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Localizer.cpp
@@ -10,9 +10,10 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/GlobalISel/Localizer.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"
#define DEBUG_TYPE "localizer"
@@ -28,7 +29,11 @@ INITIALIZE_PASS_END(Localizer, DEBUG_TYPE,
"Move/duplicate certain instructions close to their use",
false, false)
-Localizer::Localizer() : MachineFunctionPass(ID) { }
+Localizer::Localizer(std::function<bool(const MachineFunction &)> F)
+ : MachineFunctionPass(ID), DoNotRunPass(F) {}
+
+Localizer::Localizer()
+ : Localizer([](const MachineFunction &) { return false; }) {}
void Localizer::init(MachineFunction &MF) {
MRI = &MF.getRegInfo();
@@ -211,6 +216,10 @@ bool Localizer::runOnMachineFunction(MachineFunction &MF) {
MachineFunctionProperties::Property::FailedISel))
return false;
+ // Don't run the pass if the target asked so.
+ if (DoNotRunPass(MF))
+ return false;
+
LLVM_DEBUG(dbgs() << "Localize instructions for: " << MF.getName() << '\n');
init(MF);
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index df770f6664ca..67d9dacda61b 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -219,19 +219,19 @@ void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0,
assert((Res == Op0) && "type mismatch");
}
-MachineInstrBuilder MachineIRBuilder::buildGEP(const DstOp &Res,
- const SrcOp &Op0,
- const SrcOp &Op1) {
+MachineInstrBuilder MachineIRBuilder::buildPtrAdd(const DstOp &Res,
+ const SrcOp &Op0,
+ const SrcOp &Op1) {
assert(Res.getLLTTy(*getMRI()).isPointer() &&
Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
assert(Op1.getLLTTy(*getMRI()).isScalar() && "invalid offset type");
- return buildInstr(TargetOpcode::G_GEP, {Res}, {Op0, Op1});
+ return buildInstr(TargetOpcode::G_PTR_ADD, {Res}, {Op0, Op1});
}
Optional<MachineInstrBuilder>
-MachineIRBuilder::materializeGEP(Register &Res, Register Op0,
- const LLT &ValueTy, uint64_t Value) {
+MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
+ const LLT &ValueTy, uint64_t Value) {
assert(Res == 0 && "Res is a result argument");
assert(ValueTy.isScalar() && "invalid offset type");
@@ -242,7 +242,7 @@ MachineIRBuilder::materializeGEP(Register &Res, Register Op0,
Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0));
auto Cst = buildConstant(ValueTy, Value);
- return buildGEP(Res, Op0, Cst.getReg(0));
+ return buildPtrAdd(Res, Op0, Cst.getReg(0));
}
MachineInstrBuilder MachineIRBuilder::buildPtrMask(const DstOp &Res,
@@ -698,8 +698,9 @@ MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
}
MachineInstrBuilder MachineIRBuilder::buildFPTrunc(const DstOp &Res,
- const SrcOp &Op) {
- return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op);
+ const SrcOp &Op,
+ Optional<unsigned> Flags) {
+ return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op, Flags);
}
MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index f0e35c65c53b..98e48f5fc1d5 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -32,6 +32,7 @@
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
+#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/BlockFrequency.h"
#include "llvm/Support/CommandLine.h"
@@ -118,16 +119,16 @@ bool RegBankSelect::assignmentMatch(
return false;
const RegisterBank *CurRegBank = RBI->getRegBank(Reg, *MRI, *TRI);
- const RegisterBank *DesiredRegBrank = ValMapping.BreakDown[0].RegBank;
+ const RegisterBank *DesiredRegBank = ValMapping.BreakDown[0].RegBank;
// Reg is free of assignment, a simple assignment will make the
// register bank to match.
OnlyAssign = CurRegBank == nullptr;
LLVM_DEBUG(dbgs() << "Does assignment already match: ";
if (CurRegBank) dbgs() << *CurRegBank; else dbgs() << "none";
dbgs() << " against ";
- assert(DesiredRegBrank && "The mapping must be valid");
- dbgs() << *DesiredRegBrank << '\n';);
- return CurRegBank == DesiredRegBrank;
+ assert(DesiredRegBank && "The mapping must be valid");
+ dbgs() << *DesiredRegBank << '\n';);
+ return CurRegBank == DesiredRegBank;
}
bool RegBankSelect::repairReg(
@@ -259,11 +260,11 @@ uint64_t RegBankSelect::getRepairCost(
return RBI->getBreakDownCost(ValMapping, CurRegBank);
if (IsSameNumOfValues) {
- const RegisterBank *DesiredRegBrank = ValMapping.BreakDown[0].RegBank;
+ const RegisterBank *DesiredRegBank = ValMapping.BreakDown[0].RegBank;
// If we repair a definition, swap the source and destination for
// the repairing.
if (MO.isDef())
- std::swap(CurRegBank, DesiredRegBrank);
+ std::swap(CurRegBank, DesiredRegBank);
// TODO: It may be possible to actually avoid the copy.
// If we repair something where the source is defined by a copy
// and the source of that copy is on the right bank, we can reuse
@@ -275,7 +276,7 @@ uint64_t RegBankSelect::getRepairCost(
// into a new virtual register.
// We would also need to propagate this information in the
// repairing placement.
- unsigned Cost = RBI->copyCost(*DesiredRegBrank, *CurRegBank,
+ unsigned Cost = RBI->copyCost(*DesiredRegBank, *CurRegBank,
RBI->getSizeInBits(MO.getReg(), *MRI, *TRI));
// TODO: use a dedicated constant for ImpossibleCost.
if (Cost != std::numeric_limits<unsigned>::max())
diff --git a/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp b/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp
index 3fcc55286beb..255ea693b5c4 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp
@@ -82,15 +82,18 @@ bool RegisterBankInfo::verify(const TargetRegisterInfo &TRI) const {
const RegisterBank *
RegisterBankInfo::getRegBank(Register Reg, const MachineRegisterInfo &MRI,
const TargetRegisterInfo &TRI) const {
- if (Register::isPhysicalRegister(Reg))
- return &getRegBankFromRegClass(getMinimalPhysRegClass(Reg, TRI));
+ if (Register::isPhysicalRegister(Reg)) {
+ // FIXME: This was probably a copy to a virtual register that does have a
+ // type we could use.
+ return &getRegBankFromRegClass(getMinimalPhysRegClass(Reg, TRI), LLT());
+ }
assert(Reg && "NoRegister does not have a register bank");
const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
if (auto *RB = RegClassOrBank.dyn_cast<const RegisterBank *>())
return RB;
if (auto *RC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>())
- return &getRegBankFromRegClass(*RC);
+ return &getRegBankFromRegClass(*RC, MRI.getType(Reg));
return nullptr;
}
@@ -108,15 +111,18 @@ RegisterBankInfo::getMinimalPhysRegClass(Register Reg,
const RegisterBank *RegisterBankInfo::getRegBankFromConstraints(
const MachineInstr &MI, unsigned OpIdx, const TargetInstrInfo &TII,
- const TargetRegisterInfo &TRI) const {
+ const MachineRegisterInfo &MRI) const {
+ const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+
// The mapping of the registers may be available via the
// register class constraints.
- const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx, &TII, &TRI);
+ const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx, &TII, TRI);
if (!RC)
return nullptr;
- const RegisterBank &RegBank = getRegBankFromRegClass(*RC);
+ Register Reg = MI.getOperand(OpIdx).getReg();
+ const RegisterBank &RegBank = getRegBankFromRegClass(*RC, MRI.getType(Reg));
// Sanity check that the target properly implemented getRegBankFromRegClass.
assert(RegBank.covers(*RC) &&
"The mapping of the register bank does not make sense");
@@ -195,7 +201,7 @@ RegisterBankInfo::getInstrMappingImpl(const MachineInstr &MI) const {
if (!CurRegBank) {
// If this is a target specific instruction, we can deduce
// the register bank from the encoding constraints.
- CurRegBank = getRegBankFromConstraints(MI, OpIdx, TII, TRI);
+ CurRegBank = getRegBankFromConstraints(MI, OpIdx, TII, MRI);
if (!CurRegBank) {
// All our attempts failed, give up.
CompleteMapping = false;
@@ -444,7 +450,7 @@ void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) {
continue;
}
if (!MO.getReg()) {
- LLVM_DEBUG(dbgs() << " is %%noreg, nothing to be done\n");
+ LLVM_DEBUG(dbgs() << " is $noreg, nothing to be done\n");
continue;
}
assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns !=
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 45618d7992ad..eeec2a5d536a 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -431,20 +431,3 @@ Optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode, const unsigned Op1,
void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
AU.addPreserved<StackProtector>();
}
-
-MVT llvm::getMVTForLLT(LLT Ty) {
- if (!Ty.isVector())
- return MVT::getIntegerVT(Ty.getSizeInBits());
-
- return MVT::getVectorVT(
- MVT::getIntegerVT(Ty.getElementType().getSizeInBits()),
- Ty.getNumElements());
-}
-
-LLT llvm::getLLTForMVT(MVT Ty) {
- if (!Ty.isVector())
- return LLT::scalar(Ty.getSizeInBits());
-
- return LLT::vector(Ty.getVectorNumElements(),
- Ty.getVectorElementType().getSizeInBits());
-}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-05 05:24:39 +0000