diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 624 |
1 files changed, 456 insertions, 168 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 35abd990f968..a1cf4cbbee1b 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -78,6 +78,7 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsWebAssembly.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/PrintPasses.h" #include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" @@ -113,6 +114,7 @@ using namespace llvm; #define DEBUG_TYPE "isel" +#define ISEL_DUMP_DEBUG_TYPE DEBUG_TYPE "-dump" STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on"); STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected"); @@ -180,6 +182,19 @@ static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false, ViewSchedDAGs = false, ViewSUnitDAGs = false; #endif +#ifndef NDEBUG +#define ISEL_DUMP(X) \ + do { \ + if (llvm::DebugFlag && \ + (isCurrentDebugType(DEBUG_TYPE) || \ + (isCurrentDebugType(ISEL_DUMP_DEBUG_TYPE) && MatchFilterFuncName))) { \ + X; \ + } \ + } while (false) +#else +#define ISEL_DUMP(X) do { } while (false) +#endif + //===---------------------------------------------------------------------===// /// /// RegisterScheduler class - Track the registration of instruction schedulers. @@ -204,6 +219,16 @@ static RegisterScheduler defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler); +static bool dontUseFastISelFor(const Function &Fn) { + // Don't enable FastISel for functions with swiftasync Arguments. + // Debug info on those is reliant on good Argument lowering, and FastISel is + // not capable of lowering the entire function. Mixing the two selectors tend + // to result in poor lowering of Arguments. + return any_of(Fn.args(), [](const Argument &Arg) { + return Arg.hasAttribute(Attribute::AttrKind::SwiftAsync); + }); +} + namespace llvm { //===--------------------------------------------------------------------===// @@ -211,29 +236,31 @@ namespace llvm { /// the optimization level on a per-function basis. class OptLevelChanger { SelectionDAGISel &IS; - CodeGenOpt::Level SavedOptLevel; + CodeGenOptLevel SavedOptLevel; bool SavedFastISel; public: - OptLevelChanger(SelectionDAGISel &ISel, - CodeGenOpt::Level NewOptLevel) : IS(ISel) { + OptLevelChanger(SelectionDAGISel &ISel, CodeGenOptLevel NewOptLevel) + : IS(ISel) { SavedOptLevel = IS.OptLevel; SavedFastISel = IS.TM.Options.EnableFastISel; - if (NewOptLevel == SavedOptLevel) - return; - IS.OptLevel = NewOptLevel; - IS.TM.setOptLevel(NewOptLevel); - LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function " - << IS.MF->getFunction().getName() << "\n"); - LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O" - << NewOptLevel << "\n"); - if (NewOptLevel == CodeGenOpt::None) { - IS.TM.setFastISel(IS.TM.getO0WantsFastISel()); - LLVM_DEBUG( - dbgs() << "\tFastISel is " - << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled") - << "\n"); + if (NewOptLevel != SavedOptLevel) { + IS.OptLevel = NewOptLevel; + IS.TM.setOptLevel(NewOptLevel); + LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function " + << IS.MF->getFunction().getName() << "\n"); + LLVM_DEBUG(dbgs() << "\tBefore: -O" << static_cast<int>(SavedOptLevel) + << " ; After: -O" << static_cast<int>(NewOptLevel) + << "\n"); + if (NewOptLevel == CodeGenOptLevel::None) + IS.TM.setFastISel(IS.TM.getO0WantsFastISel()); } + if (dontUseFastISelFor(IS.MF->getFunction())) + IS.TM.setFastISel(false); + LLVM_DEBUG( + dbgs() << "\tFastISel is " + << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled") + << "\n"); } ~OptLevelChanger() { @@ -241,8 +268,8 @@ namespace llvm { return; LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function " << IS.MF->getFunction().getName() << "\n"); - LLVM_DEBUG(dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O" - << SavedOptLevel << "\n"); + LLVM_DEBUG(dbgs() << "\tBefore: -O" << static_cast<int>(IS.OptLevel) + << " ; After: -O" << static_cast<int>(SavedOptLevel) << "\n"); IS.OptLevel = SavedOptLevel; IS.TM.setOptLevel(SavedOptLevel); IS.TM.setFastISel(SavedFastISel); @@ -252,8 +279,8 @@ namespace llvm { //===--------------------------------------------------------------------===// /// createDefaultScheduler - This creates an instruction scheduler appropriate /// for the target. - ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS, - CodeGenOpt::Level OptLevel) { + ScheduleDAGSDNodes *createDefaultScheduler(SelectionDAGISel *IS, + CodeGenOptLevel OptLevel) { const TargetLowering *TLI = IS->TLI; const TargetSubtargetInfo &ST = IS->MF->getSubtarget(); @@ -262,7 +289,7 @@ namespace llvm { return SchedulerCtor(IS, OptLevel); } - if (OptLevel == CodeGenOpt::None || + if (OptLevel == CodeGenOptLevel::None || (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) || TLI->getSchedulingPreference() == Sched::Source) return createSourceListDAGScheduler(IS, OptLevel); @@ -315,7 +342,7 @@ void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, //===----------------------------------------------------------------------===// SelectionDAGISel::SelectionDAGISel(char &ID, TargetMachine &tm, - CodeGenOpt::Level OL) + CodeGenOptLevel OL) : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()), SwiftError(new SwiftErrorValueTracking()), CurDAG(new SelectionDAG(tm, OL)), @@ -335,23 +362,23 @@ SelectionDAGISel::~SelectionDAGISel() { } void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { - if (OptLevel != CodeGenOpt::None) - AU.addRequired<AAResultsWrapperPass>(); + if (OptLevel != CodeGenOptLevel::None) + AU.addRequired<AAResultsWrapperPass>(); AU.addRequired<GCModuleInfo>(); AU.addRequired<StackProtector>(); AU.addPreserved<GCModuleInfo>(); AU.addRequired<TargetLibraryInfoWrapperPass>(); AU.addRequired<TargetTransformInfoWrapperPass>(); AU.addRequired<AssumptionCacheTracker>(); - if (UseMBPI && OptLevel != CodeGenOpt::None) - AU.addRequired<BranchProbabilityInfoWrapperPass>(); + if (UseMBPI && OptLevel != CodeGenOptLevel::None) + AU.addRequired<BranchProbabilityInfoWrapperPass>(); AU.addRequired<ProfileSummaryInfoWrapperPass>(); // AssignmentTrackingAnalysis only runs if assignment tracking is enabled for // the module. AU.addRequired<AssignmentTrackingAnalysis>(); AU.addPreserved<AssignmentTrackingAnalysis>(); - if (OptLevel != CodeGenOpt::None) - LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); + if (OptLevel != CodeGenOptLevel::None) + LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -391,6 +418,13 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { const Function &Fn = mf.getFunction(); MF = &mf; +#ifndef NDEBUG + StringRef FuncName = Fn.getName(); + MatchFilterFuncName = isFunctionInPrintList(FuncName); +#else + (void)MatchFilterFuncName; +#endif + // Decide what flavour of variable location debug-info will be used, before // we change the optimisation level. bool InstrRef = mf.shouldUseDebugInstrRef(); @@ -403,9 +437,9 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // it wants to look at it. TM.resetTargetOptions(Fn); // Reset OptLevel to None for optnone functions. - CodeGenOpt::Level NewOptLevel = OptLevel; - if (OptLevel != CodeGenOpt::None && skipFunction(Fn)) - NewOptLevel = CodeGenOpt::None; + CodeGenOptLevel NewOptLevel = OptLevel; + if (OptLevel != CodeGenOptLevel::None && skipFunction(Fn)) + NewOptLevel = CodeGenOptLevel::None; OptLevelChanger OLC(*this, NewOptLevel); TII = MF->getSubtarget().getInstrInfo(); @@ -417,14 +451,14 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.getFunction()); auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); BlockFrequencyInfo *BFI = nullptr; - if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOpt::None) + if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOptLevel::None) BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI(); FunctionVarLocs const *FnVarLocs = nullptr; if (isAssignmentTrackingEnabled(*Fn.getParent())) FnVarLocs = getAnalysis<AssignmentTrackingAnalysis>().getResults(); - LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); + ISEL_DUMP(dbgs() << "\n\n\n=== " << FuncName << "\n"); UniformityInfo *UA = nullptr; if (auto *UAPass = getAnalysisIfAvailable<UniformityInfoWrapperPass>()) @@ -438,12 +472,12 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // into account). That's unfortunate but OK because it just means we won't // ask for passes that have been required anyway. - if (UseMBPI && OptLevel != CodeGenOpt::None) + if (UseMBPI && OptLevel != CodeGenOptLevel::None) FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI(); else FuncInfo->BPI = nullptr; - if (OptLevel != CodeGenOpt::None) + if (OptLevel != CodeGenOptLevel::None) AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); else AA = nullptr; @@ -456,7 +490,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // We split CSR if the target supports it for the given function // and the function has only return exits. - if (OptLevel != CodeGenOpt::None && TLI->supportSplitCSR(MF)) { + if (OptLevel != CodeGenOptLevel::None && TLI->supportSplitCSR(MF)) { FuncInfo->SplitCSR = true; // Collect all the return blocks. @@ -656,8 +690,8 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // at this point. FuncInfo->clear(); - LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n"); - LLVM_DEBUG(MF->print(dbgs())); + ISEL_DUMP(dbgs() << "*** MachineFunction at end of ISel ***\n"); + ISEL_DUMP(MF->print(dbgs())); return true; } @@ -685,10 +719,13 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, CurDAG->NewNodesMustHaveLegalTypes = false; // Lower the instructions. If a call is emitted as a tail call, cease emitting - // nodes for this block. + // nodes for this block. If an instruction is elided, don't emit it, but do + // handle any debug-info attached to it. for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) { if (!ElidedArgCopyInstrs.count(&*I)) SDB->visit(*I); + else + SDB->visitDbgInfo(*I); } // Make sure the root of the DAG is up-to-date. @@ -765,10 +802,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { BlockName = (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str(); } - LLVM_DEBUG(dbgs() << "Initial selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nInitial selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -785,10 +822,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->Combine(BeforeLegalizeTypes, AA, OptLevel); } - LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nOptimized lowered selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -807,10 +844,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { Changed = CurDAG->LegalizeTypes(); } - LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nType-legalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -831,10 +868,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->Combine(AfterLegalizeTypes, AA, OptLevel); } - LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nOptimized type-legalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -849,10 +886,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { } if (Changed) { - LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nVector-legalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -865,10 +902,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->LegalizeTypes(); } - LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nVector/type-legalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -885,10 +922,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->Combine(AfterLegalizeVectorOps, AA, OptLevel); } - LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nOptimized vector-legalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -905,10 +942,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->Legalize(); } - LLVM_DEBUG(dbgs() << "Legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nLegalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) @@ -925,17 +962,17 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { CurDAG->Combine(AfterLegalizeDAG, AA, OptLevel); } - LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nOptimized legalized selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); #ifndef NDEBUG if (TTI.hasBranchDivergence()) CurDAG->VerifyDAGDivergence(); #endif - if (OptLevel != CodeGenOpt::None) + if (OptLevel != CodeGenOptLevel::None) ComputeLiveOutVRegInfo(); if (ViewISelDAGs && MatchFilterBB) @@ -949,10 +986,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { DoInstructionSelection(); } - LLVM_DEBUG(dbgs() << "Selected selection DAG: " - << printMBBReference(*FuncInfo->MBB) << " '" << BlockName - << "'\n"; - CurDAG->dump()); + ISEL_DUMP(dbgs() << "\nSelected selection DAG: " + << printMBBReference(*FuncInfo->MBB) << " '" << BlockName + << "'\n"; + CurDAG->dump()); if (ViewSchedDAGs && MatchFilterBB) CurDAG->viewGraph("scheduler input for " + BlockName); @@ -1357,6 +1394,8 @@ static bool processIfEntryValueDbgDeclare(FunctionLoweringInfo &FuncInfo, // Find the corresponding livein physical register to this argument. for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins()) if (VirtReg == ArgVReg) { + // Append an op deref to account for the fact that this is a dbg_declare. + Expr = DIExpression::append(Expr, dwarf::DW_OP_deref); FuncInfo.MF->setVariableDbgInfo(Var, Expr, PhysReg, DbgLoc); LLVM_DEBUG(dbgs() << "processDbgDeclare: setVariableDbgInfo Var=" << *Var << ", Expr=" << *Expr << ", MCRegister=" << PhysReg @@ -1422,6 +1461,14 @@ static void processDbgDeclares(FunctionLoweringInfo &FuncInfo) { if (DI && processDbgDeclare(FuncInfo, DI->getAddress(), DI->getExpression(), DI->getVariable(), DI->getDebugLoc())) FuncInfo.PreprocessedDbgDeclares.insert(DI); + + for (const DPValue &DPV : I.getDbgValueRange()) { + if (DPV.getType() == DPValue::LocationType::Declare && + processDbgDeclare(FuncInfo, DPV.getVariableLocationOp(0), + DPV.getExpression(), DPV.getVariable(), + DPV.getDebugLoc())) + FuncInfo.PreprocessedDPVDeclares.insert(&DPV); + } } } @@ -1510,7 +1557,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Iterate over all basic blocks in the function. StackProtector &SP = getAnalysis<StackProtector>(); for (const BasicBlock *LLVMBB : RPOT) { - if (OptLevel != CodeGenOpt::None) { + if (OptLevel != CodeGenOptLevel::None) { bool AllPredsVisited = true; for (const BasicBlock *Pred : predecessors(LLVMBB)) { if (!FuncInfo->VisitedBBs.count(Pred)) { @@ -2074,41 +2121,43 @@ void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, --e; // Don't process a glue operand if it is here. while (i != e) { - unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue(); - if (!InlineAsm::isMemKind(Flags) && !InlineAsm::isFuncKind(Flags)) { + InlineAsm::Flag Flags(cast<ConstantSDNode>(InOps[i])->getZExtValue()); + if (!Flags.isMemKind() && !Flags.isFuncKind()) { // Just skip over this operand, copying the operands verbatim. - Ops.insert(Ops.end(), InOps.begin()+i, - InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1); - i += InlineAsm::getNumOperandRegisters(Flags) + 1; + Ops.insert(Ops.end(), InOps.begin() + i, + InOps.begin() + i + Flags.getNumOperandRegisters() + 1); + i += Flags.getNumOperandRegisters() + 1; } else { - assert(InlineAsm::getNumOperandRegisters(Flags) == 1 && + assert(Flags.getNumOperandRegisters() == 1 && "Memory operand with multiple values?"); unsigned TiedToOperand; - if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) { + if (Flags.isUseOperandTiedToDef(TiedToOperand)) { // We need the constraint ID from the operand this is tied to. unsigned CurOp = InlineAsm::Op_FirstOperand; - Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue(); + Flags = + InlineAsm::Flag(cast<ConstantSDNode>(InOps[CurOp])->getZExtValue()); for (; TiedToOperand; --TiedToOperand) { - CurOp += InlineAsm::getNumOperandRegisters(Flags)+1; - Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue(); + CurOp += Flags.getNumOperandRegisters() + 1; + Flags = InlineAsm::Flag( + cast<ConstantSDNode>(InOps[CurOp])->getZExtValue()); } } // Otherwise, this is a memory operand. Ask the target to select it. std::vector<SDValue> SelOps; - unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags); + const InlineAsm::ConstraintCode ConstraintID = + Flags.getMemoryConstraintID(); if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps)) report_fatal_error("Could not match memory address. Inline asm" " failure!"); // Add this to the output node. - unsigned NewFlags = - InlineAsm::isMemKind(Flags) - ? InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size()) - : InlineAsm::getFlagWord(InlineAsm::Kind_Func, SelOps.size()); - NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID); - Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32)); + Flags = InlineAsm::Flag(Flags.isMemKind() ? InlineAsm::Kind::Mem + : InlineAsm::Kind::Func, + SelOps.size()); + Flags.setMemConstraint(ConstraintID); + Ops.push_back(CurDAG->getTargetConstant(Flags, DL, MVT::i32)); llvm::append_range(Ops, SelOps); i += 2; } @@ -2176,18 +2225,20 @@ static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, /// operand node N of U during instruction selection that starts at Root. bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { - if (OptLevel == CodeGenOpt::None) return false; + if (OptLevel == CodeGenOptLevel::None) + return false; return N.hasOneUse(); } /// IsLegalToFold - Returns true if the specific operand node N of /// U can be folded during instruction selection that starts at Root. bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, - CodeGenOpt::Level OptLevel, + CodeGenOptLevel OptLevel, bool IgnoreChains) { - if (OptLevel == CodeGenOpt::None) return false; + if (OptLevel == CodeGenOptLevel::None) + return false; - // If Root use can somehow reach N through a path that that doesn't contain + // If Root use can somehow reach N through a path that doesn't contain // U then folding N would create a cycle. e.g. In the following // diagram, Root can reach N through X. If N is folded into Root, then // X is both a predecessor and a successor of U. @@ -2435,6 +2486,13 @@ GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { return Val; } +void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(SDNode *N) { + SDLoc dl(N); + CurDAG->SelectNodeTo(N, TargetOpcode::JUMP_TABLE_DEBUG_INFO, MVT::Glue, + CurDAG->getTargetConstant(N->getConstantOperandVal(1), + dl, MVT::i64, true)); +} + /// When a match is complete, this method updates uses of interior chain results /// to use the new results. void SelectionDAGISel::UpdateChains( @@ -2591,7 +2649,7 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, unsigned ResNumResults = Res->getNumValues(); // Move the glue if needed. if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 && - (unsigned)OldGlueResultNo != ResNumResults-1) + static_cast<unsigned>(OldGlueResultNo) != ResNumResults - 1) ReplaceUses(SDValue(Node, OldGlueResultNo), SDValue(Res, ResNumResults - 1)); @@ -2600,7 +2658,7 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, // Move the chain reference if needed. if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 && - (unsigned)OldChainResultNo != ResNumResults-1) + static_cast<unsigned>(OldChainResultNo) != ResNumResults - 1) ReplaceUses(SDValue(Node, OldChainResultNo), SDValue(Res, ResNumResults - 1)); @@ -2639,8 +2697,11 @@ LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckChildSame( /// CheckPatternPredicate - Implements OP_CheckPatternPredicate. LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, - const SelectionDAGISel &SDISel) { - return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]); + const SelectionDAGISel &SDISel, bool TwoBytePredNo) { + unsigned PredNo = MatcherTable[MatcherIndex++]; + if (TwoBytePredNo) + PredNo |= MatcherTable[MatcherIndex++] << 8; + return SDISel.CheckPatternPredicate(PredNo); } /// CheckNodePredicate - Implements OP_CheckNodePredicate. @@ -2654,35 +2715,34 @@ LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N) { uint16_t Opc = MatcherTable[MatcherIndex++]; - Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; + Opc |= static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8; return N->getOpcode() == Opc; } -LLVM_ATTRIBUTE_ALWAYS_INLINE static bool -CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, - const TargetLowering *TLI, const DataLayout &DL) { - MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; - if (N.getValueType() == VT) return true; +LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckType(MVT::SimpleValueType VT, + SDValue N, + const TargetLowering *TLI, + const DataLayout &DL) { + if (N.getValueType() == VT) + return true; // Handle the case when VT is iPTR. return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL); } LLVM_ATTRIBUTE_ALWAYS_INLINE static bool -CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, - SDValue N, const TargetLowering *TLI, const DataLayout &DL, - unsigned ChildNo) { +CheckChildType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, + const DataLayout &DL, unsigned ChildNo) { if (ChildNo >= N.getNumOperands()) - return false; // Match fails if out of range child #. - return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI, - DL); + return false; // Match fails if out of range child #. + return ::CheckType(VT, N.getOperand(ChildNo), TLI, DL); } LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N) { return cast<CondCodeSDNode>(N)->get() == - (ISD::CondCode)MatcherTable[MatcherIndex++]; + static_cast<ISD::CondCode>(MatcherTable[MatcherIndex++]); } LLVM_ATTRIBUTE_ALWAYS_INLINE static bool @@ -2696,7 +2756,8 @@ CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL) { - MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + MVT::SimpleValueType VT = + static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); if (cast<VTSDNode>(N)->getVT() == VT) return true; @@ -2773,7 +2834,8 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) { - switch (Table[Index++]) { + unsigned Opcode = Table[Index++]; + switch (Opcode) { default: Result = false; return Index-1; // Could not evaluate this predicate. @@ -2788,7 +2850,10 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same); return Index; case SelectionDAGISel::OPC_CheckPatternPredicate: - Result = !::CheckPatternPredicate(Table, Index, SDISel); + case SelectionDAGISel::OPC_CheckPatternPredicate2: + Result = !::CheckPatternPredicate( + Table, Index, SDISel, + Table[Index - 1] == SelectionDAGISel::OPC_CheckPatternPredicate2); return Index; case SelectionDAGISel::OPC_CheckPredicate: Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode()); @@ -2797,12 +2862,27 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, Result = !::CheckOpcode(Table, Index, N.getNode()); return Index; case SelectionDAGISel::OPC_CheckType: - Result = !::CheckType(Table, Index, N, SDISel.TLI, - SDISel.CurDAG->getDataLayout()); + case SelectionDAGISel::OPC_CheckTypeI32: + case SelectionDAGISel::OPC_CheckTypeI64: { + MVT::SimpleValueType VT; + switch (Opcode) { + case SelectionDAGISel::OPC_CheckTypeI32: + VT = MVT::i32; + break; + case SelectionDAGISel::OPC_CheckTypeI64: + VT = MVT::i64; + break; + default: + VT = static_cast<MVT::SimpleValueType>(Table[Index++]); + break; + } + Result = !::CheckType(VT, N, SDISel.TLI, SDISel.CurDAG->getDataLayout()); return Index; + } case SelectionDAGISel::OPC_CheckTypeRes: { unsigned Res = Table[Index++]; - Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI, + Result = !::CheckType(static_cast<MVT::SimpleValueType>(Table[Index++]), + N.getValue(Res), SDISel.TLI, SDISel.CurDAG->getDataLayout()); return Index; } @@ -2814,10 +2894,40 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, case SelectionDAGISel::OPC_CheckChild5Type: case SelectionDAGISel::OPC_CheckChild6Type: case SelectionDAGISel::OPC_CheckChild7Type: - Result = !::CheckChildType( - Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(), - Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type); + case SelectionDAGISel::OPC_CheckChild0TypeI32: + case SelectionDAGISel::OPC_CheckChild1TypeI32: + case SelectionDAGISel::OPC_CheckChild2TypeI32: + case SelectionDAGISel::OPC_CheckChild3TypeI32: + case SelectionDAGISel::OPC_CheckChild4TypeI32: + case SelectionDAGISel::OPC_CheckChild5TypeI32: + case SelectionDAGISel::OPC_CheckChild6TypeI32: + case SelectionDAGISel::OPC_CheckChild7TypeI32: + case SelectionDAGISel::OPC_CheckChild0TypeI64: + case SelectionDAGISel::OPC_CheckChild1TypeI64: + case SelectionDAGISel::OPC_CheckChild2TypeI64: + case SelectionDAGISel::OPC_CheckChild3TypeI64: + case SelectionDAGISel::OPC_CheckChild4TypeI64: + case SelectionDAGISel::OPC_CheckChild5TypeI64: + case SelectionDAGISel::OPC_CheckChild6TypeI64: + case SelectionDAGISel::OPC_CheckChild7TypeI64: { + MVT::SimpleValueType VT; + unsigned ChildNo; + if (Opcode >= SelectionDAGISel::OPC_CheckChild0TypeI32 && + Opcode <= SelectionDAGISel::OPC_CheckChild7TypeI32) { + VT = MVT::i32; + ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeI32; + } else if (Opcode >= SelectionDAGISel::OPC_CheckChild0TypeI64 && + Opcode <= SelectionDAGISel::OPC_CheckChild7TypeI64) { + VT = MVT::i64; + ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeI64; + } else { + VT = static_cast<MVT::SimpleValueType>(Table[Index++]); + ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0Type; + } + Result = !::CheckChildType(VT, N, SDISel.TLI, + SDISel.CurDAG->getDataLayout(), ChildNo); return Index; + } case SelectionDAGISel::OPC_CheckCondCode: Result = !::CheckCondCode(Table, Index, N); return Index; @@ -2981,6 +3091,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::PATCHPOINT: Select_PATCHPOINT(NodeToMatch); return; + case ISD::JUMP_TABLE_DEBUG_INFO: + Select_JUMP_TABLE_DEBUG_INFO(NodeToMatch); + return; } assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); @@ -3042,7 +3155,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // Get the opcode, add the index to the table. uint16_t Opc = MatcherTable[Idx++]; - Opc |= (unsigned short)MatcherTable[Idx++] << 8; + Opc |= static_cast<uint16_t>(MatcherTable[Idx++]) << 8; if (Opc >= OpcodeOffset.size()) OpcodeOffset.resize((Opc+1)*2); OpcodeOffset[Opc] = Idx; @@ -3059,7 +3172,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, #ifndef NDEBUG unsigned CurrentOpcodeIndex = MatcherIndex; #endif - BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++]; + BuiltinOpcodes Opcode = + static_cast<BuiltinOpcodes>(MatcherTable[MatcherIndex++]); switch (Opcode) { case OPC_Scope: { // Okay, the semantics of this operation are that we should push a scope @@ -3179,6 +3293,29 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; } + case OPC_MoveSibling: + case OPC_MoveSibling0: + case OPC_MoveSibling1: + case OPC_MoveSibling2: + case OPC_MoveSibling3: + case OPC_MoveSibling4: + case OPC_MoveSibling5: + case OPC_MoveSibling6: + case OPC_MoveSibling7: { + // Pop the current node off the NodeStack. + NodeStack.pop_back(); + assert(!NodeStack.empty() && "Node stack imbalance!"); + N = NodeStack.back(); + + unsigned SiblingNo = Opcode == OPC_MoveSibling + ? MatcherTable[MatcherIndex++] + : Opcode - OPC_MoveSibling0; + if (SiblingNo >= N.getNumOperands()) + break; // Match fails if out of range sibling #. + N = N.getOperand(SiblingNo); + NodeStack.push_back(N); + continue; + } case OPC_MoveParent: // Pop the current node off the NodeStack. NodeStack.pop_back(); @@ -3198,7 +3335,10 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; case OPC_CheckPatternPredicate: - if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break; + case OPC_CheckPatternPredicate2: + if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this, + Opcode == OPC_CheckPatternPredicate2)) + break; continue; case OPC_CheckPredicate: if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this, @@ -3240,15 +3380,29 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; case OPC_CheckType: - if (!::CheckType(MatcherTable, MatcherIndex, N, TLI, - CurDAG->getDataLayout())) + case OPC_CheckTypeI32: + case OPC_CheckTypeI64: + MVT::SimpleValueType VT; + switch (Opcode) { + case OPC_CheckTypeI32: + VT = MVT::i32; + break; + case OPC_CheckTypeI64: + VT = MVT::i64; + break; + default: + VT = static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); + break; + } + if (!::CheckType(VT, N, TLI, CurDAG->getDataLayout())) break; continue; case OPC_CheckTypeRes: { unsigned Res = MatcherTable[MatcherIndex++]; - if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI, - CurDAG->getDataLayout())) + if (!::CheckType( + static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]), + N.getValue(Res), TLI, CurDAG->getDataLayout())) break; continue; } @@ -3265,7 +3419,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, if (CaseSize == 0) break; uint16_t Opc = MatcherTable[MatcherIndex++]; - Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; + Opc |= static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8; // If the opcode matches, then we will execute this case. if (CurNodeOpcode == Opc) @@ -3295,7 +3449,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex); if (CaseSize == 0) break; - MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + MVT CaseVT = + static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); if (CaseVT == MVT::iPTR) CaseVT = TLI->getPointerTy(CurDAG->getDataLayout()); @@ -3316,15 +3471,48 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, << '\n'); continue; } - case OPC_CheckChild0Type: case OPC_CheckChild1Type: - case OPC_CheckChild2Type: case OPC_CheckChild3Type: - case OPC_CheckChild4Type: case OPC_CheckChild5Type: - case OPC_CheckChild6Type: case OPC_CheckChild7Type: - if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI, - CurDAG->getDataLayout(), - Opcode - OPC_CheckChild0Type)) + case OPC_CheckChild0Type: + case OPC_CheckChild1Type: + case OPC_CheckChild2Type: + case OPC_CheckChild3Type: + case OPC_CheckChild4Type: + case OPC_CheckChild5Type: + case OPC_CheckChild6Type: + case OPC_CheckChild7Type: + case OPC_CheckChild0TypeI32: + case OPC_CheckChild1TypeI32: + case OPC_CheckChild2TypeI32: + case OPC_CheckChild3TypeI32: + case OPC_CheckChild4TypeI32: + case OPC_CheckChild5TypeI32: + case OPC_CheckChild6TypeI32: + case OPC_CheckChild7TypeI32: + case OPC_CheckChild0TypeI64: + case OPC_CheckChild1TypeI64: + case OPC_CheckChild2TypeI64: + case OPC_CheckChild3TypeI64: + case OPC_CheckChild4TypeI64: + case OPC_CheckChild5TypeI64: + case OPC_CheckChild6TypeI64: + case OPC_CheckChild7TypeI64: { + MVT::SimpleValueType VT; + unsigned ChildNo; + if (Opcode >= SelectionDAGISel::OPC_CheckChild0TypeI32 && + Opcode <= SelectionDAGISel::OPC_CheckChild7TypeI32) { + VT = MVT::i32; + ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeI32; + } else if (Opcode >= SelectionDAGISel::OPC_CheckChild0TypeI64 && + Opcode <= SelectionDAGISel::OPC_CheckChild7TypeI64) { + VT = MVT::i64; + ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeI64; + } else { + VT = static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); + ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0Type; + } + if (!::CheckChildType(VT, N, TLI, CurDAG->getDataLayout(), ChildNo)) break; continue; + } case OPC_CheckCondCode: if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break; continue; @@ -3390,22 +3578,43 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; } case OPC_EmitInteger: - case OPC_EmitStringInteger: { - MVT::SimpleValueType VT = - (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + case OPC_EmitInteger8: + case OPC_EmitInteger16: + case OPC_EmitInteger32: + case OPC_EmitInteger64: + case OPC_EmitStringInteger: + case OPC_EmitStringInteger32: { + MVT::SimpleValueType VT; + switch (Opcode) { + case OPC_EmitInteger8: + VT = MVT::i8; + break; + case OPC_EmitInteger16: + VT = MVT::i16; + break; + case OPC_EmitInteger32: + case OPC_EmitStringInteger32: + VT = MVT::i32; + break; + case OPC_EmitInteger64: + VT = MVT::i64; + break; + default: + VT = static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); + break; + } int64_t Val = MatcherTable[MatcherIndex++]; if (Val & 128) Val = GetVBR(Val, MatcherTable, MatcherIndex); - if (Opcode == OPC_EmitInteger) + if (Opcode >= OPC_EmitInteger && Opcode <= OPC_EmitInteger64) Val = decodeSignRotatedValue(Val); - RecordedNodes.push_back(std::pair<SDValue, SDNode*>( - CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch), - VT), nullptr)); + RecordedNodes.push_back(std::pair<SDValue, SDNode *>( + CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch), VT), nullptr)); continue; } case OPC_EmitRegister: { MVT::SimpleValueType VT = - (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); unsigned RegNo = MatcherTable[MatcherIndex++]; RecordedNodes.push_back(std::pair<SDValue, SDNode*>( CurDAG->getRegister(RegNo, VT), nullptr)); @@ -3416,7 +3625,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // values are stored in two bytes in the matcher table (just like // opcodes). MVT::SimpleValueType VT = - (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); unsigned RegNo = MatcherTable[MatcherIndex++]; RegNo |= MatcherTable[MatcherIndex++] << 8; RecordedNodes.push_back(std::pair<SDValue, SDNode*>( @@ -3424,9 +3633,19 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; } - case OPC_EmitConvertToTarget: { + case OPC_EmitConvertToTarget: + case OPC_EmitConvertToTarget0: + case OPC_EmitConvertToTarget1: + case OPC_EmitConvertToTarget2: + case OPC_EmitConvertToTarget3: + case OPC_EmitConvertToTarget4: + case OPC_EmitConvertToTarget5: + case OPC_EmitConvertToTarget6: + case OPC_EmitConvertToTarget7: { // Convert from IMM/FPIMM to target version. - unsigned RecNo = MatcherTable[MatcherIndex++]; + unsigned RecNo = Opcode == OPC_EmitConvertToTarget + ? MatcherTable[MatcherIndex++] + : Opcode - OPC_EmitConvertToTarget0; assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget"); SDValue Imm = RecordedNodes[RecNo].first; @@ -3522,11 +3741,22 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, } case OPC_EmitCopyToReg: - case OPC_EmitCopyToReg2: { - unsigned RecNo = MatcherTable[MatcherIndex++]; + case OPC_EmitCopyToReg0: + case OPC_EmitCopyToReg1: + case OPC_EmitCopyToReg2: + case OPC_EmitCopyToReg3: + case OPC_EmitCopyToReg4: + case OPC_EmitCopyToReg5: + case OPC_EmitCopyToReg6: + case OPC_EmitCopyToReg7: + case OPC_EmitCopyToRegTwoByte: { + unsigned RecNo = + Opcode >= OPC_EmitCopyToReg0 && Opcode <= OPC_EmitCopyToReg7 + ? Opcode - OPC_EmitCopyToReg0 + : MatcherTable[MatcherIndex++]; assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg"); unsigned DestPhysReg = MatcherTable[MatcherIndex++]; - if (Opcode == OPC_EmitCopyToReg2) + if (Opcode == OPC_EmitCopyToRegTwoByte) DestPhysReg |= MatcherTable[MatcherIndex++] << 8; if (!InputChain.getNode()) @@ -3558,26 +3788,83 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, continue; } - case OPC_EmitNode: case OPC_MorphNodeTo: - case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2: - case OPC_MorphNodeTo0: case OPC_MorphNodeTo1: case OPC_MorphNodeTo2: { + case OPC_EmitNode: + case OPC_EmitNode0: + case OPC_EmitNode1: + case OPC_EmitNode2: + case OPC_EmitNode0None: + case OPC_EmitNode1None: + case OPC_EmitNode2None: + case OPC_EmitNode0Chain: + case OPC_EmitNode1Chain: + case OPC_EmitNode2Chain: + case OPC_MorphNodeTo: + case OPC_MorphNodeTo0: + case OPC_MorphNodeTo1: + case OPC_MorphNodeTo2: + case OPC_MorphNodeTo0None: + case OPC_MorphNodeTo1None: + case OPC_MorphNodeTo2None: + case OPC_MorphNodeTo0Chain: + case OPC_MorphNodeTo1Chain: + case OPC_MorphNodeTo2Chain: + case OPC_MorphNodeTo0GlueInput: + case OPC_MorphNodeTo1GlueInput: + case OPC_MorphNodeTo2GlueInput: + case OPC_MorphNodeTo0GlueOutput: + case OPC_MorphNodeTo1GlueOutput: + case OPC_MorphNodeTo2GlueOutput: { uint16_t TargetOpc = MatcherTable[MatcherIndex++]; - TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; - unsigned EmitNodeInfo = MatcherTable[MatcherIndex++]; + TargetOpc |= static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8; + unsigned EmitNodeInfo; + if (Opcode >= OPC_EmitNode0None && Opcode <= OPC_EmitNode2Chain) { + if (Opcode >= OPC_EmitNode0Chain && Opcode <= OPC_EmitNode2Chain) + EmitNodeInfo = OPFL_Chain; + else + EmitNodeInfo = OPFL_None; + } else if (Opcode >= OPC_MorphNodeTo0None && + Opcode <= OPC_MorphNodeTo2GlueOutput) { + if (Opcode >= OPC_MorphNodeTo0Chain && Opcode <= OPC_MorphNodeTo2Chain) + EmitNodeInfo = OPFL_Chain; + else if (Opcode >= OPC_MorphNodeTo0GlueInput && + Opcode <= OPC_MorphNodeTo2GlueInput) + EmitNodeInfo = OPFL_GlueInput; + else if (Opcode >= OPC_MorphNodeTo0GlueOutput && + Opcode <= OPC_MorphNodeTo2GlueOutput) + EmitNodeInfo = OPFL_GlueOutput; + else + EmitNodeInfo = OPFL_None; + } else + EmitNodeInfo = MatcherTable[MatcherIndex++]; // Get the result VT list. unsigned NumVTs; // If this is one of the compressed forms, get the number of VTs based // on the Opcode. Otherwise read the next byte from the table. if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2) NumVTs = Opcode - OPC_MorphNodeTo0; + else if (Opcode >= OPC_MorphNodeTo0None && Opcode <= OPC_MorphNodeTo2None) + NumVTs = Opcode - OPC_MorphNodeTo0None; + else if (Opcode >= OPC_MorphNodeTo0Chain && + Opcode <= OPC_MorphNodeTo2Chain) + NumVTs = Opcode - OPC_MorphNodeTo0Chain; + else if (Opcode >= OPC_MorphNodeTo0GlueInput && + Opcode <= OPC_MorphNodeTo2GlueInput) + NumVTs = Opcode - OPC_MorphNodeTo0GlueInput; + else if (Opcode >= OPC_MorphNodeTo0GlueOutput && + Opcode <= OPC_MorphNodeTo2GlueOutput) + NumVTs = Opcode - OPC_MorphNodeTo0GlueOutput; else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2) NumVTs = Opcode - OPC_EmitNode0; + else if (Opcode >= OPC_EmitNode0None && Opcode <= OPC_EmitNode2None) + NumVTs = Opcode - OPC_EmitNode0None; + else if (Opcode >= OPC_EmitNode0Chain && Opcode <= OPC_EmitNode2Chain) + NumVTs = Opcode - OPC_EmitNode0Chain; else NumVTs = MatcherTable[MatcherIndex++]; SmallVector<EVT, 4> VTs; for (unsigned i = 0; i != NumVTs; ++i) { MVT::SimpleValueType VT = - (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; + static_cast<MVT::SimpleValueType>(MatcherTable[MatcherIndex++]); if (VT == MVT::iPTR) VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy; VTs.push_back(VT); @@ -3644,8 +3931,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // Create the node. MachineSDNode *Res = nullptr; - bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo || - (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2); + bool IsMorphNodeTo = + Opcode == OPC_MorphNodeTo || + (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2GlueOutput); if (!IsMorphNodeTo) { // If this is a normal EmitNode command, just create the new node and // add the results to the RecordedNodes list. @@ -3667,7 +3955,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, auto &Chain = ChainNodesMatched; assert((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"); - llvm::erase_value(Chain, N); + llvm::erase(Chain, N); }); Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo)); |