aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp')
-rw-r--r--llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp194
1 files changed, 126 insertions, 68 deletions
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index b97c369b832d..73b763710fdf 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -72,6 +72,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetIntrinsicInfo.h"
#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/MemoryOpRemark.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -97,6 +98,7 @@ INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(StackProtector)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
false, false)
@@ -164,6 +166,8 @@ void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<GISelCSEAnalysisWrapperPass>();
if (OptLevel != CodeGenOpt::None)
AU.addRequired<BranchProbabilityInfoWrapperPass>();
+ AU.addRequired<TargetLibraryInfoWrapperPass>();
+ AU.addPreserved<TargetLibraryInfoWrapperPass>();
getSelectionDAGFallbackAnalysisUsage(AU);
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -253,23 +257,13 @@ int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
Align IRTranslator::getMemOpAlign(const Instruction &I) {
if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
return SI->getAlign();
- if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
+ if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
return LI->getAlign();
- }
- if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
- // TODO(PR27168): This instruction has no alignment attribute, but unlike
- // the default alignment for load/store, the default here is to assume
- // it has NATURAL alignment, not DataLayout-specified alignment.
- const DataLayout &DL = AI->getModule()->getDataLayout();
- return Align(DL.getTypeStoreSize(AI->getCompareOperand()->getType()));
- }
- if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
- // TODO(PR27168): This instruction has no alignment attribute, but unlike
- // the default alignment for load/store, the default here is to assume
- // it has NATURAL alignment, not DataLayout-specified alignment.
- const DataLayout &DL = AI->getModule()->getDataLayout();
- return Align(DL.getTypeStoreSize(AI->getValOperand()->getType()));
- }
+ if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
+ return AI->getAlign();
+ if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
+ return AI->getAlign();
+
OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
R << "unable to translate memop: " << ore::NV("Opcode", &I);
reportTranslationError(*MF, *TPC, *ORE, R);
@@ -840,9 +834,8 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
// For conditional branch lowering, we might try to do something silly like
// emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
// just re-use the existing condition vreg.
- if (CI && CI->getZExtValue() == 1 &&
- MRI->getType(CondLHS).getSizeInBits() == 1 &&
- CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
+ if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI &&
+ CI->getZExtValue() == 1 && CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
Cond = CondLHS;
} else {
Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
@@ -1307,7 +1300,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
AAMDNodes AAMetadata;
LI.getAAMetadata(AAMetadata);
auto MMO = MF->getMachineMemOperand(
- Ptr, Flags, MRI->getType(Regs[i]).getSizeInBytes(),
+ Ptr, Flags, MRI->getType(Regs[i]),
commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges,
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
@@ -1349,7 +1342,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
AAMDNodes AAMetadata;
SI.getAAMetadata(AAMetadata);
auto MMO = MF->getMachineMemOperand(
- Ptr, Flags, MRI->getType(Vals[i]).getSizeInBytes(),
+ Ptr, Flags, MRI->getType(Vals[i]),
commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr,
SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
@@ -1479,14 +1472,22 @@ bool IRTranslator::translateGetElementPtr(const User &U,
// Normalize Vector GEP - all scalar operands should be converted to the
// splat vector.
unsigned VectorWidth = 0;
- if (auto *VT = dyn_cast<VectorType>(U.getType()))
+
+ // True if we should use a splat vector; using VectorWidth alone is not
+ // sufficient.
+ bool WantSplatVector = false;
+ if (auto *VT = dyn_cast<VectorType>(U.getType())) {
VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
+ // We don't produce 1 x N vectors; those are treated as scalars.
+ WantSplatVector = VectorWidth > 1;
+ }
// We might need to splat the base pointer into a vector if the offsets
// are vectors.
- if (VectorWidth && !PtrTy.isVector()) {
+ if (WantSplatVector && !PtrTy.isVector()) {
BaseReg =
- MIRBuilder.buildSplatVector(LLT::vector(VectorWidth, PtrTy), BaseReg)
+ MIRBuilder
+ .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg)
.getReg(0);
PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
PtrTy = getLLTForType(*PtrIRTy, *DL);
@@ -1522,7 +1523,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
Register IdxReg = getOrCreateVReg(*Idx);
LLT IdxTy = MRI->getType(IdxReg);
if (IdxTy != OffsetTy) {
- if (!IdxTy.isVector() && VectorWidth) {
+ if (!IdxTy.isVector() && WantSplatVector) {
IdxReg = MIRBuilder.buildSplatVector(
OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
}
@@ -1571,7 +1572,7 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
Register SrcReg = getOrCreateVReg(**AI);
LLT SrcTy = MRI->getType(SrcReg);
if (SrcTy.isPointer())
- MinPtrSize = std::min(SrcTy.getSizeInBits(), MinPtrSize);
+ MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
SrcRegs.push_back(SrcReg);
}
@@ -1595,6 +1596,9 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
DstAlign = MCI->getDestAlign().valueOrOne();
SrcAlign = MCI->getSourceAlign().valueOrOne();
+ } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
+ DstAlign = MCI->getDestAlign().valueOrOne();
+ SrcAlign = MCI->getSourceAlign().valueOrOne();
} else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
DstAlign = MMI->getDestAlign().valueOrOne();
SrcAlign = MMI->getSourceAlign().valueOrOne();
@@ -1603,10 +1607,12 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
DstAlign = MSI->getDestAlign().valueOrOne();
}
- // We need to propagate the tail call flag from the IR inst as an argument.
- // Otherwise, we have to pessimize and assume later that we cannot tail call
- // any memory intrinsics.
- ICall.addImm(CI.isTailCall() ? 1 : 0);
+ if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
+ // We need to propagate the tail call flag from the IR inst as an argument.
+ // Otherwise, we have to pessimize and assume later that we cannot tail call
+ // any memory intrinsics.
+ ICall.addImm(CI.isTailCall() ? 1 : 0);
+ }
// Create mem operands to store the alignment and volatile info.
auto VolFlag = IsVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
@@ -1633,12 +1639,14 @@ void IRTranslator::getStackGuard(Register DstReg,
if (!Global)
return;
+ unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
+ LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
+
MachinePointerInfo MPInfo(Global);
auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
MachineMemOperand::MODereferenceable;
- MachineMemOperand *MemRef =
- MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
- DL->getPointerABIAlignment(0));
+ MachineMemOperand *MemRef = MF->getMachineMemOperand(
+ MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
MIB.setMemRefs({MemRef});
}
@@ -1826,6 +1834,16 @@ bool IRTranslator::translateConstrainedFPIntrinsic(
bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
+ if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
+ if (ORE->enabled()) {
+ const Function &F = *MI->getParent()->getParent();
+ auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+ if (MemoryOpRemark::canHandle(MI, TLI)) {
+ MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
+ R.visit(MI);
+ }
+ }
+ }
// If this is a simple intrinsic (that is, we just need to add a def of
// a vreg, and uses for each arg operand, then translate it.
@@ -1924,9 +1942,9 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
assert(DI.getVariable()->isValidLocationForIntrinsic(
MIRBuilder.getDebugLoc()) &&
"Expected inlined-at fields to agree");
- if (!V) {
- // Currently the optimizer can produce this; insert an undef to
- // help debugging. Probably the optimizer should not do this.
+ if (!V || DI.hasArgList()) {
+ // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
+ // terminate any prior location.
MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
} else if (const auto *CI = dyn_cast<Constant>(V)) {
MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
@@ -2027,6 +2045,8 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
getOrCreateVReg(*CI.getArgOperand(0)),
MachineInstr::copyFlagsFromInstruction(CI));
return true;
+ case Intrinsic::memcpy_inline:
+ return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
case Intrinsic::memcpy:
return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
case Intrinsic::memmove:
@@ -2063,7 +2083,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
*MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
MachineMemOperand::MOStore |
MachineMemOperand::MOVolatile,
- PtrTy.getSizeInBits() / 8, Align(8)));
+ PtrTy, Align(8)));
return true;
}
case Intrinsic::stacksave: {
@@ -2255,6 +2275,17 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
Args.push_back(getOrCreateVRegs(*Arg));
}
+ if (auto *CI = dyn_cast<CallInst>(&CB)) {
+ if (ORE->enabled()) {
+ const Function &F = *CI->getParent()->getParent();
+ auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+ if (MemoryOpRemark::canHandle(CI, TLI)) {
+ MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
+ R.visit(CI);
+ }
+ }
+ }
+
// We don't set HasCalls on MFI here yet because call lowering may decide to
// optimize into tail calls. Instead, we defer that to selection where a final
// scan is done to check if any instructions are calls.
@@ -2349,10 +2380,11 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
Align Alignment = Info.align.getValueOr(
DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
-
- uint64_t Size = Info.memVT.getStoreSize();
+ LLT MemTy = Info.memVT.isSimple()
+ ? getLLTForMVT(Info.memVT.getSimpleVT())
+ : LLT::scalar(Info.memVT.getStoreSizeInBits());
MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
- Info.flags, Size, Alignment));
+ Info.flags, MemTy, Alignment));
}
return true;
@@ -2423,8 +2455,6 @@ bool IRTranslator::translateInvoke(const User &U,
const BasicBlock *EHPadBB = I.getSuccessor(1);
const Function *Fn = I.getCalledFunction();
- if (I.isInlineAsm())
- return false;
// FIXME: support invoking patchpoint and statepoint intrinsics.
if (Fn && Fn->isIntrinsic())
@@ -2442,12 +2472,37 @@ bool IRTranslator::translateInvoke(const User &U,
if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
return false;
+ bool LowerInlineAsm = false;
+ if (I.isInlineAsm()) {
+ const InlineAsm *IA = cast<InlineAsm>(I.getCalledOperand());
+ if (!IA->canThrow()) {
+ // Fast path without emitting EH_LABELs.
+
+ if (!translateInlineAsm(I, MIRBuilder))
+ return false;
+
+ MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB(),
+ *ReturnMBB = &getMBB(*ReturnBB);
+
+ // Update successor info.
+ addSuccessorWithProb(InvokeMBB, ReturnMBB, BranchProbability::getOne());
+
+ MIRBuilder.buildBr(*ReturnMBB);
+ return true;
+ } else {
+ LowerInlineAsm = true;
+ }
+ }
+
// Emit the actual call, bracketed by EH_LABELs so that the MF knows about
// the region covered by the try.
MCSymbol *BeginSymbol = Context.createTempSymbol();
MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
- if (!translateCallBase(I, MIRBuilder))
+ if (LowerInlineAsm) {
+ if (!translateInlineAsm(I, MIRBuilder))
+ return false;
+ } else if (!translateCallBase(I, MIRBuilder))
return false;
MCSymbol *EndSymbol = Context.createTempSymbol();
@@ -2695,9 +2750,6 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
auto &TLI = *MF->getSubtarget().getTargetLowering();
auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
- Type *ResType = I.getType();
- Type *ValType = ResType->Type::getStructElementType(0);
-
auto Res = getOrCreateVRegs(I);
Register OldValRes = Res[0];
Register SuccessRes = Res[1];
@@ -2711,9 +2763,9 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
MIRBuilder.buildAtomicCmpXchgWithSuccess(
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(
- MachinePointerInfo(I.getPointerOperand()), Flags,
- DL->getTypeStoreSize(ValType), getMemOpAlign(I), AAMetadata, nullptr,
- I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
+ MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
+ getMemOpAlign(I), AAMetadata, nullptr, I.getSyncScopeID(),
+ I.getSuccessOrdering(), I.getFailureOrdering()));
return true;
}
@@ -2723,8 +2775,6 @@ bool IRTranslator::translateAtomicRMW(const User &U,
auto &TLI = *MF->getSubtarget().getTargetLowering();
auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
- Type *ResType = I.getType();
-
Register Res = getOrCreateVReg(I);
Register Addr = getOrCreateVReg(*I.getPointerOperand());
Register Val = getOrCreateVReg(*I.getValOperand());
@@ -2780,9 +2830,9 @@ bool IRTranslator::translateAtomicRMW(const User &U,
MIRBuilder.buildAtomicRMW(
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
- Flags, DL->getTypeStoreSize(ResType),
- getMemOpAlign(I), AAMetadata, nullptr,
- I.getSyncScopeID(), I.getOrdering()));
+ Flags, MRI->getType(Val), getMemOpAlign(I),
+ AAMetadata, nullptr, I.getSyncScopeID(),
+ I.getOrdering()));
return true;
}
@@ -2853,13 +2903,6 @@ bool IRTranslator::valueIsSplit(const Value &V,
bool IRTranslator::translate(const Instruction &Inst) {
CurBuilder->setDebugLoc(Inst.getDebugLoc());
- // We only emit constants into the entry block from here. To prevent jumpy
- // debug behaviour set the line to 0.
- if (const DebugLoc &DL = Inst.getDebugLoc())
- EntryBuilder->setDebugLoc(DILocation::get(
- Inst.getContext(), 0, 0, DL.getScope(), DL.getInlinedAt()));
- else
- EntryBuilder->setDebugLoc(DebugLoc());
auto &TLI = *MF->getSubtarget().getTargetLowering();
if (TLI.fallBackToDAGISel(Inst))
@@ -2876,6 +2919,13 @@ bool IRTranslator::translate(const Instruction &Inst) {
}
bool IRTranslator::translate(const Constant &C, Register Reg) {
+ // We only emit constants into the entry block from here. To prevent jumpy
+ // debug behaviour set the line to 0.
+ if (auto CurrInstDL = CurBuilder->getDL())
+ EntryBuilder->setDebugLoc(DILocation::get(C.getContext(), 0, 0,
+ CurrInstDL.getScope(),
+ CurrInstDL.getInlinedAt()));
+
if (auto CI = dyn_cast<ConstantInt>(&C))
EntryBuilder->buildConstant(Reg, *CI);
else if (auto CF = dyn_cast<ConstantFP>(&C))
@@ -2887,14 +2937,15 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
else if (auto GV = dyn_cast<GlobalValue>(&C))
EntryBuilder->buildGlobalValue(Reg, GV);
else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
- if (!CAZ->getType()->isVectorTy())
+ if (!isa<FixedVectorType>(CAZ->getType()))
return false;
// Return the scalar if it is a <1 x Ty> vector.
- if (CAZ->getNumElements() == 1)
+ unsigned NumElts = CAZ->getElementCount().getFixedValue();
+ if (NumElts == 1)
return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get());
SmallVector<Register, 4> Ops;
- for (unsigned i = 0; i < CAZ->getNumElements(); ++i) {
- Constant &Elt = *CAZ->getElementValue(i);
+ for (unsigned I = 0; I < NumElts; ++I) {
+ Constant &Elt = *CAZ->getElementValue(I);
Ops.push_back(getOrCreateVReg(Elt));
}
EntryBuilder->buildBuildVector(Reg, Ops);
@@ -2968,8 +3019,13 @@ void IRTranslator::finalizeBasicBlock() {
emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
- // FIXME delete this block below?
if (BTB.ContiguousRange && j + 2 == ej) {
+ // We need to record the replacement phi edge here that normally
+ // happens in emitBitTestCase before we delete the case, otherwise the
+ // phi edge will be lost.
+ addMachineCFGPred({BTB.Parent->getBasicBlock(),
+ BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
+ MBB);
// Since we're not going to use the final bit test, remove it.
BTB.Cases.pop_back();
break;
@@ -3079,7 +3135,9 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
assert(PendingPHIs.empty() && "stale PHIs");
- if (!DL->isLittleEndian()) {
+ // Targets which want to use big endian can enable it using
+ // enableBigEndian()
+ if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
// Currently we don't properly handle big endian code.
OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
F.getSubprogram(), &F.getEntryBlock());
@@ -3121,7 +3179,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
// Make our arguments/constants entry block fallthrough to the IR entry block.
EntryBB->addSuccessor(&getMBB(F.front()));
- if (CLI->fallBackToDAGISel(F)) {
+ if (CLI->fallBackToDAGISel(*MF)) {
OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
F.getSubprogram(), &F.getEntryBlock());
R << "unable to lower function: " << ore::NV("Prototype", F.getType());
generated by cgit v1.3 (git 2.53.0) at 2026-09-03 15:56:26 +0000