summaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp209
1 files changed, 137 insertions, 72 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 1f07a241a824..6c57c72d47a7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -27,8 +27,10 @@
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/EHPersonalities.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/FastISel.h"
@@ -71,10 +73,12 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Pass.h"
@@ -147,17 +151,17 @@ static cl::opt<bool>
ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before legalize types"));
static cl::opt<bool>
-ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
- cl::desc("Pop up a window to show dags before legalize"));
+ ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
+ cl::desc("Pop up a window to show dags before the post "
+ "legalize types dag combine pass"));
+static cl::opt<bool>
+ ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
+ cl::desc("Pop up a window to show dags before legalize"));
static cl::opt<bool>
ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before the second "
"dag combine pass"));
static cl::opt<bool>
-ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
- cl::desc("Pop up a window to show dags before the post legalize types"
- " dag combine pass"));
-static cl::opt<bool>
ViewISelDAGs("view-isel-dags", cl::Hidden,
cl::desc("Pop up a window to show isel dags as they are selected"));
static cl::opt<bool>
@@ -167,12 +171,10 @@ static cl::opt<bool>
ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
cl::desc("Pop up a window to show SUnit dags after they are processed"));
#else
-static const bool ViewDAGCombine1 = false,
- ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false,
- ViewDAGCombine2 = false,
- ViewDAGCombineLT = false,
- ViewISelDAGs = false, ViewSchedDAGs = false,
- ViewSUnitDAGs = false;
+static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false,
+ ViewDAGCombineLT = false, ViewLegalizeDAGs = false,
+ ViewDAGCombine2 = false, ViewISelDAGs = false,
+ ViewSchedDAGs = false, ViewSUnitDAGs = false;
#endif
//===---------------------------------------------------------------------===//
@@ -305,28 +307,22 @@ void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
// SelectionDAGISel code
//===----------------------------------------------------------------------===//
-SelectionDAGISel::SelectionDAGISel(TargetMachine &tm,
- CodeGenOpt::Level OL) :
- MachineFunctionPass(ID), TM(tm),
- FuncInfo(new FunctionLoweringInfo()),
- SwiftError(new SwiftErrorValueTracking()),
- CurDAG(new SelectionDAG(tm, OL)),
- SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, *SwiftError, OL)),
- AA(), GFI(),
- OptLevel(OL),
- DAGSize(0) {
- initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());
- initializeBranchProbabilityInfoWrapperPassPass(
- *PassRegistry::getPassRegistry());
- initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
- initializeTargetLibraryInfoWrapperPassPass(
- *PassRegistry::getPassRegistry());
- }
+SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL)
+ : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()),
+ SwiftError(new SwiftErrorValueTracking()),
+ CurDAG(new SelectionDAG(tm, OL)),
+ SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError,
+ OL)),
+ AA(), GFI(), OptLevel(OL), DAGSize(0) {
+ initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());
+ initializeBranchProbabilityInfoWrapperPassPass(
+ *PassRegistry::getPassRegistry());
+ initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
+ initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
+}
SelectionDAGISel::~SelectionDAGISel() {
- delete SDB;
delete CurDAG;
- delete FuncInfo;
delete SwiftError;
}
@@ -340,6 +336,8 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<TargetTransformInfoWrapperPass>();
if (UseMBPI && OptLevel != CodeGenOpt::None)
AU.addRequired<BranchProbabilityInfoWrapperPass>();
+ AU.addRequired<ProfileSummaryInfoWrapperPass>();
+ LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -442,13 +440,17 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
LoopInfo *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
+ auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
+ auto *BFI = (PSI && PSI->hasProfileSummary()) ?
+ &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() :
+ nullptr;
LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n");
SplitCriticalSideEffectEdges(const_cast<Function &>(Fn), DT, LI);
CurDAG->init(*MF, *ORE, this, LibInfo,
- getAnalysisIfAvailable<LegacyDivergenceAnalysis>());
+ getAnalysisIfAvailable<LegacyDivergenceAnalysis>(), PSI, BFI);
FuncInfo->set(Fn, *MF, CurDAG);
SwiftError->setFunction(*MF);
@@ -735,23 +737,20 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
}
void SelectionDAGISel::ComputeLiveOutVRegInfo() {
- SmallPtrSet<SDNode*, 16> VisitedNodes;
+ SmallPtrSet<SDNode *, 16> Added;
SmallVector<SDNode*, 128> Worklist;
Worklist.push_back(CurDAG->getRoot().getNode());
+ Added.insert(CurDAG->getRoot().getNode());
KnownBits Known;
do {
SDNode *N = Worklist.pop_back_val();
- // If we've already seen this node, ignore it.
- if (!VisitedNodes.insert(N).second)
- continue;
-
// Otherwise, add all chain operands to the worklist.
for (const SDValue &Op : N->op_values())
- if (Op.getValueType() == MVT::Other)
+ if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second)
Worklist.push_back(Op.getNode());
// If this is a CopyToReg with a vreg dest, process it.
@@ -793,8 +792,8 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
FuncInfo->MBB->getBasicBlock()->getName());
#endif
#ifdef NDEBUG
- if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||
- ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs ||
+ if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewDAGCombineLT ||
+ ViewLegalizeDAGs || ViewDAGCombine2 || ViewISelDAGs || ViewSchedDAGs ||
ViewSUnitDAGs)
#endif
{
@@ -1159,10 +1158,30 @@ void SelectionDAGISel::DoInstructionSelection() {
// we convert them to normal FP opcodes instead at this point. This
// will allow them to be handled by existing target-specific instruction
// selectors.
- if (Node->isStrictFPOpcode() &&
- (TLI->getOperationAction(Node->getOpcode(), Node->getValueType(0))
- != TargetLowering::Legal))
- Node = CurDAG->mutateStrictFPToFP(Node);
+ if (!TLI->isStrictFPEnabled() && Node->isStrictFPOpcode()) {
+ // For some opcodes, we need to call TLI->getOperationAction using
+ // the first operand type instead of the result type. Note that this
+ // must match what SelectionDAGLegalize::LegalizeOp is doing.
+ EVT ActionVT;
+ switch (Node->getOpcode()) {
+ case ISD::STRICT_SINT_TO_FP:
+ case ISD::STRICT_UINT_TO_FP:
+ case ISD::STRICT_LRINT:
+ case ISD::STRICT_LLRINT:
+ case ISD::STRICT_LROUND:
+ case ISD::STRICT_LLROUND:
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS:
+ ActionVT = Node->getOperand(1).getValueType();
+ break;
+ default:
+ ActionVT = Node->getValueType(0);
+ break;
+ }
+ if (TLI->getOperationAction(Node->getOpcode(), ActionVT)
+ == TargetLowering::Expand)
+ Node = CurDAG->mutateStrictFPToFP(Node);
+ }
LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";
Node->dump(CurDAG));
@@ -1280,20 +1299,20 @@ bool SelectionDAGISel::PrepareEHLandingPad() {
/// side-effect free and is either dead or folded into a generated instruction.
/// Return false if it needs to be emitted.
static bool isFoldedOrDeadInstruction(const Instruction *I,
- FunctionLoweringInfo *FuncInfo) {
+ const FunctionLoweringInfo &FuncInfo) {
return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.
!I->isTerminator() && // Terminators aren't folded.
- !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
- !I->isEHPad() && // EH pad instructions aren't folded.
- !FuncInfo->isExportedInst(I); // Exported instrs must be computed.
+ !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.
+ !I->isEHPad() && // EH pad instructions aren't folded.
+ !FuncInfo.isExportedInst(I); // Exported instrs must be computed.
}
/// Collect llvm.dbg.declare information. This is done after argument lowering
/// in case the declarations refer to arguments.
-static void processDbgDeclares(FunctionLoweringInfo *FuncInfo) {
- MachineFunction *MF = FuncInfo->MF;
+static void processDbgDeclares(FunctionLoweringInfo &FuncInfo) {
+ MachineFunction *MF = FuncInfo.MF;
const DataLayout &DL = MF->getDataLayout();
- for (const BasicBlock &BB : *FuncInfo->Fn) {
+ for (const BasicBlock &BB : *FuncInfo.Fn) {
for (const Instruction &I : BB) {
const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I);
if (!DI)
@@ -1315,11 +1334,11 @@ static void processDbgDeclares(FunctionLoweringInfo *FuncInfo) {
// intrinsic and handle this during isel like dbg.value.
int FI = std::numeric_limits<int>::max();
if (const auto *AI = dyn_cast<AllocaInst>(Address)) {
- auto SI = FuncInfo->StaticAllocaMap.find(AI);
- if (SI != FuncInfo->StaticAllocaMap.end())
+ auto SI = FuncInfo.StaticAllocaMap.find(AI);
+ if (SI != FuncInfo.StaticAllocaMap.end())
FI = SI->second;
} else if (const auto *Arg = dyn_cast<Argument>(Address))
- FI = FuncInfo->getArgumentFrameIndex(Arg);
+ FI = FuncInfo.getArgumentFrameIndex(Arg);
if (FI == std::numeric_limits<int>::max())
continue;
@@ -1353,7 +1372,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];
FuncInfo->InsertPt = FuncInfo->MBB->begin();
- CurDAG->setFunctionLoweringInfo(FuncInfo);
+ CurDAG->setFunctionLoweringInfo(FuncInfo.get());
if (!FastIS) {
LowerArguments(Fn);
@@ -1393,7 +1412,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
if (FastIS && Inserted)
FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));
- processDbgDeclares(FuncInfo);
+ processDbgDeclares(*FuncInfo);
// Iterate over all basic blocks in the function.
StackProtector &SP = getAnalysis<StackProtector>();
@@ -1453,7 +1472,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
const Instruction *Inst = &*std::prev(BI);
// If we no longer require this instruction, skip it.
- if (isFoldedOrDeadInstruction(Inst, FuncInfo) ||
+ if (isFoldedOrDeadInstruction(Inst, *FuncInfo) ||
ElidedArgCopyInstrs.count(Inst)) {
--NumFastIselRemaining;
continue;
@@ -1473,7 +1492,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
const Instruction *BeforeInst = Inst;
while (BeforeInst != &*Begin) {
BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));
- if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo))
+ if (!isFoldedOrDeadInstruction(BeforeInst, *FuncInfo))
break;
}
if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
@@ -1589,7 +1608,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
// But if FastISel was run, we already selected some of the block.
// If we emitted a tail-call, we need to delete any previously emitted
// instruction that follows it.
- if (HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
+ if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());
}
@@ -2230,10 +2249,13 @@ void SelectionDAGISel::Select_INLINEASM(SDNode *N, bool Branch) {
void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
SDLoc dl(Op);
- MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
- const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
+ MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
+ const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
+
+ EVT VT = Op->getValueType(0);
+ LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
Register Reg =
- TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
+ TLI->getRegisterByName(RegStr->getString().data(), Ty,
CurDAG->getMachineFunction());
SDValue New = CurDAG->getCopyFromReg(
Op->getOperand(0), dl, Reg, Op->getValueType(0));
@@ -2244,10 +2266,13 @@ void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
SDLoc dl(Op);
- MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
- const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
- Register Reg = TLI->getRegisterByName(RegStr->getString().data(),
- Op->getOperand(2).getValueType(),
+ MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1));
+ const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0));
+
+ EVT VT = Op->getOperand(2).getValueType();
+ LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
+
+ Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty,
CurDAG->getMachineFunction());
SDValue New = CurDAG->getCopyToReg(
Op->getOperand(0), dl, Reg, Op->getOperand(2));
@@ -3176,13 +3201,19 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
case OPC_CheckFoldableChainNode: {
assert(NodeStack.size() != 1 && "No parent node");
// Verify that all intermediate nodes between the root and this one have
- // a single use.
+ // a single use (ignoring chains, which are handled in UpdateChains).
bool HasMultipleUses = false;
- for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i)
- if (!NodeStack[i].getNode()->hasOneUse()) {
- HasMultipleUses = true;
- break;
- }
+ for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) {
+ unsigned NNonChainUses = 0;
+ SDNode *NS = NodeStack[i].getNode();
+ for (auto UI = NS->use_begin(), UE = NS->use_end(); UI != UE; ++UI)
+ if (UI.getUse().getValueType() != MVT::Other)
+ if (++NNonChainUses > 1) {
+ HasMultipleUses = true;
+ break;
+ }
+ if (HasMultipleUses) break;
+ }
if (HasMultipleUses) break;
// Check to see that the target thinks this is profitable to fold and that
@@ -3433,6 +3464,17 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)
Ops.push_back(InputGlue);
+ // Check whether any matched node could raise an FP exception. Since all
+ // such nodes must have a chain, it suffices to check ChainNodesMatched.
+ // We need to perform this check before potentially modifying one of the
+ // nodes via MorphNode.
+ bool MayRaiseFPException = false;
+ for (auto *N : ChainNodesMatched)
+ if (mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept()) {
+ MayRaiseFPException = true;
+ break;
+ }
+
// Create the node.
MachineSDNode *Res = nullptr;
bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||
@@ -3464,6 +3506,14 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
Ops, EmitNodeInfo));
}
+ // Set the NoFPExcept flag when no original matched node could
+ // raise an FP exception, but the new node potentially might.
+ if (!MayRaiseFPException && mayRaiseFPException(Res)) {
+ SDNodeFlags Flags = Res->getFlags();
+ Flags.setNoFPExcept(true);
+ Res->setFlags(Flags);
+ }
+
// If the node had chain/glue results, update our notion of the current
// chain and glue.
if (EmitNodeInfo & OPFL_GlueOutput) {
@@ -3619,6 +3669,21 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
}
}
+/// Return whether the node may raise an FP exception.
+bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const {
+ // For machine opcodes, consult the MCID flag.
+ if (N->isMachineOpcode()) {
+ const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
+ return MCID.mayRaiseFPException();
+ }
+
+ // For ISD opcodes, only StrictFP opcodes may raise an FP
+ // exception.
+ if (N->isTargetOpcode())
+ return N->isTargetStrictFPOpcode();
+ return N->isStrictFPOpcode();
+}
+
bool SelectionDAGISel::isOrEquivalentToAdd(const SDNode *N) const {
assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-07 13:33:07 +0000