diff options
Diffstat (limited to 'include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h')
| -rw-r--r-- | include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h | 728 |
1 files changed, 567 insertions, 161 deletions
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index 98b6b859b9e26..ac2c055ab1452 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -1,4 +1,4 @@ -//==-- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h ---------*- C++ -*-==// +//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,7 +16,32 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <cstddef> +#include <cstdint> + namespace llvm { + +/// GlobalISel PatFrag Predicates +enum { + GIPFP_I64_Invalid = 0, + GIPFP_APInt_Invalid = 0, + GIPFP_APFloat_Invalid = 0, +}; + template <class TgtInstructionSelector, class PredicateBitset, class ComplexMatcherMemFn> bool InstructionSelector::executeMatchTable( @@ -24,306 +49,687 @@ bool InstructionSelector::executeMatchTable( const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo, const int64_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI, - const PredicateBitset &AvailableFeatures) const { - const int64_t *Command = MatchTable; + const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, + CodeGenCoverage &CoverageInfo) const { + uint64_t CurrentIdx = 0; + SmallVector<uint64_t, 8> OnFailResumeAt; + + enum RejectAction { RejectAndGiveUp, RejectAndResume }; + auto handleReject = [&]() -> RejectAction { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Rejected\n"); + if (OnFailResumeAt.empty()) + return RejectAndGiveUp; + CurrentIdx = OnFailResumeAt.back(); + OnFailResumeAt.pop_back(); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" + << OnFailResumeAt.size() << " try-blocks remain)\n"); + return RejectAndResume; + }; + while (true) { - switch (*Command++) { + assert(CurrentIdx != ~0u && "Invalid MatchTable index"); + switch (MatchTable[CurrentIdx++]) { + case GIM_Try: { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Begin try-block\n"); + OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); + break; + } + case GIM_RecordInsn: { - int64_t NewInsnID = *Command++; - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; // As an optimisation we require that MIs[0] is always the root. Refuse // any attempt to modify it. assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); - (void)NewInsnID; MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); if (!MO.isReg()) { - DEBUG(dbgs() << "Rejected (not a register)\n"); - return false; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Not a register\n"); + if (handleReject() == RejectAndGiveUp) + return false; + break; } if (TRI.isPhysicalRegister(MO.getReg())) { - DEBUG(dbgs() << "Rejected (is a physical register)\n"); - return false; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Is a physical register\n"); + if (handleReject() == RejectAndGiveUp) + return false; + break; } - assert((size_t)NewInsnID == State.MIs.size() && - "Expected to store MIs in order"); - State.MIs.push_back(MRI.getVRegDef(MO.getReg())); - DEBUG(dbgs() << "MIs[" << NewInsnID << "] = GIM_RecordInsn(" << InsnID - << ", " << OpIdx << ")\n"); + MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); + if ((size_t)NewInsnID < State.MIs.size()) + State.MIs[NewInsnID] = NewMI; + else { + assert((size_t)NewInsnID == State.MIs.size() && + "Expected to store MIs in order"); + State.MIs.push_back(NewMI); + } + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": MIs[" << NewInsnID + << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx + << ")\n"); break; } case GIM_CheckFeatures: { - int64_t ExpectedBitsetID = *Command++; - DEBUG(dbgs() << "GIM_CheckFeatures(ExpectedBitsetID=" << ExpectedBitsetID - << ")\n"); + int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckFeatures(ExpectedBitsetID=" + << ExpectedBitsetID << ")\n"); if ((AvailableFeatures & MatcherInfo.FeatureBitsets[ExpectedBitsetID]) != MatcherInfo.FeatureBitsets[ExpectedBitsetID]) { - DEBUG(dbgs() << "Rejected\n"); - return false; + if (handleReject() == RejectAndGiveUp) + return false; } break; } case GIM_CheckOpcode: { - int64_t InsnID = *Command++; - int64_t Expected = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Expected = MatchTable[CurrentIdx++]; unsigned Opcode = State.MIs[InsnID]->getOpcode(); - DEBUG(dbgs() << "GIM_CheckOpcode(MIs[" << InsnID << "], ExpectedOpcode=" - << Expected << ") // Got=" << Opcode << "\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID + << "], ExpectedOpcode=" << Expected + << ") // Got=" << Opcode << "\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (Opcode != Expected) - return false; + if (Opcode != Expected) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckNumOperands: { - int64_t InsnID = *Command++; - int64_t Expected = *Command++; - DEBUG(dbgs() << "GIM_CheckNumOperands(MIs[" << InsnID - << "], Expected=" << Expected << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Expected = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" + << InsnID << "], Expected=" << Expected << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (State.MIs[InsnID]->getNumOperands() != Expected) - return false; + if (State.MIs[InsnID]->getNumOperands() != Expected) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_CheckI64ImmPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && + "Expected G_CONSTANT"); + assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); + int64_t Value = 0; + if (State.MIs[InsnID]->getOperand(1).isCImm()) + Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); + else if (State.MIs[InsnID]->getOperand(1).isImm()) + Value = State.MIs[InsnID]->getOperand(1).getImm(); + else + llvm_unreachable("Expected Imm or CImm operand"); + + if (!MatcherInfo.I64ImmPredicateFns[Predicate](Value)) + if (handleReject() == RejectAndGiveUp) + return false; break; } + case GIM_CheckAPIntImmPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[InsnID]->getOpcode() && "Expected G_CONSTANT"); + assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); + APInt Value; + if (State.MIs[InsnID]->getOperand(1).isCImm()) + Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); + else + llvm_unreachable("Expected Imm or CImm operand"); + + if (!MatcherInfo.APIntImmPredicateFns[Predicate](Value)) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAPFloatImmPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && + "Expected G_FCONSTANT"); + assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); + assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); + APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); + + if (!MatcherInfo.APFloatImmPredicateFns[Predicate](Value)) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAtomicOrdering: { + int64_t InsnID = MatchTable[CurrentIdx++]; + AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" + << InsnID << "], " << (uint64_t)Ordering << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (MMO->getOrdering() != Ordering) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAtomicOrderingOrStrongerThan: { + int64_t InsnID = MatchTable[CurrentIdx++]; + AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" + << InsnID << "], " << (uint64_t)Ordering << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAtomicOrderingWeakerThan: { + int64_t InsnID = MatchTable[CurrentIdx++]; + AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" + << InsnID << "], " << (uint64_t)Ordering << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (!isStrongerThan(Ordering, MMO->getOrdering())) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckType: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t TypeID = *Command++; - DEBUG(dbgs() << "GIM_CheckType(MIs[" << InsnID << "]->getOperand(" - << OpIdx << "), TypeID=" << TypeID << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t TypeID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID + << "]->getOperand(" << OpIdx + << "), TypeID=" << TypeID << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) != - MatcherInfo.TypeObjects[TypeID]) - return false; + MatcherInfo.TypeObjects[TypeID]) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_CheckPointerToAny: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t SizeInBits = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), SizeInBits=" << SizeInBits << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + // iPTR must be looked up in the target. + if (SizeInBits == 0) { + MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); + SizeInBits = MF->getDataLayout().getPointerSizeInBits(0); + } + + assert(SizeInBits != 0 && "Pointer size must be known"); + + const LLT &Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()); + if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } case GIM_CheckRegBankForClass: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t RCEnum = *Command++; - DEBUG(dbgs() << "GIM_CheckRegBankForClass(MIs[" << InsnID - << "]->getOperand(" << OpIdx << "), RCEnum=" << RCEnum - << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RCEnum = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), RCEnum=" << RCEnum << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); if (&RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != - RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, TRI)) - return false; + RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, + TRI)) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckComplexPattern: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t RendererID = *Command++; - int64_t ComplexPredicateID = *Command++; - DEBUG(dbgs() << "State.Renderers[" << RendererID - << "] = GIM_CheckComplexPattern(MIs[" << InsnID - << "]->getOperand(" << OpIdx - << "), ComplexPredicateID=" << ComplexPredicateID << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": State.Renderers[" << RendererID + << "] = GIM_CheckComplexPattern(MIs[" << InsnID + << "]->getOperand(" << OpIdx + << "), ComplexPredicateID=" << ComplexPredicateID + << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); // FIXME: Use std::invoke() when it's available. - if (!(State.Renderers[RendererID] = - (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( - State.MIs[InsnID]->getOperand(OpIdx)))) - return false; + ComplexRendererFns Renderer = + (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( + State.MIs[InsnID]->getOperand(OpIdx)); + if (Renderer.hasValue()) + State.Renderers[RendererID] = Renderer.getValue(); + else + if (handleReject() == RejectAndGiveUp) + return false; break; } + case GIM_CheckConstantInt: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t Value = *Command++; - DEBUG(dbgs() << "GIM_CheckConstantInt(MIs[" << InsnID << "]->getOperand(" - << OpIdx << "), Value=" << Value << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, MRI)) - return false; + + // isOperandImmEqual() will sign-extend to 64-bits, so should we. + LLT Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()); + Value = SignExtend64(Value, Ty.getSizeInBits()); + + if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, + MRI)) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckLiteralInt: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t Value = *Command++; - DEBUG(dbgs() << "GIM_CheckLiteralInt(MIs[" << InsnID << "]->getOperand(" << OpIdx - << "), Value=" << Value << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx); - if (!OM.isCImm() || !OM.getCImm()->equalsInt(Value)) - return false; + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckIntrinsicID: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t Value = *Command++; - DEBUG(dbgs() << "GIM_CheckIntrinsicID(MIs[" << InsnID << "]->getOperand(" << OpIdx - << "), Value=" << Value << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx); - if (!OM.isIntrinsicID() || OM.getIntrinsicID() != Value) - return false; + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) + if (handleReject() == RejectAndGiveUp) + return false; break; } + case GIM_CheckIsMBB: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - DEBUG(dbgs() << "GIM_CheckIsMBB(MIs[" << InsnID << "]->getOperand(" - << OpIdx << "))\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID + << "]->getOperand(" << OpIdx << "))\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) - return false; + if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } case GIM_CheckIsSafeToFold: { - int64_t InsnID = *Command++; - DEBUG(dbgs() << "GIM_CheckIsSafeToFold(MIs[" << InsnID << "])\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" + << InsnID << "])\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!isObviouslySafeToFold(*State.MIs[InsnID])) - return false; + if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_CheckIsSameOperand: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t OtherInsnID = MatchTable[CurrentIdx++]; + int64_t OtherOpIdx = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" + << InsnID << "][" << OpIdx << "], MIs[" + << OtherInsnID << "][" << OtherOpIdx << "])\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); + if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( + State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_Reject: + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_Reject"); + if (handleReject() == RejectAndGiveUp) + return false; + break; case GIR_MutateOpcode: { - int64_t OldInsnID = *Command++; - int64_t NewInsnID = *Command++; - int64_t NewOpcode = *Command++; - assert((size_t)NewInsnID == OutMIs.size() && - "Expected to store MIs in order"); - OutMIs.push_back( - MachineInstrBuilder(*State.MIs[OldInsnID]->getParent()->getParent(), - State.MIs[OldInsnID])); + int64_t OldInsnID = MatchTable[CurrentIdx++]; + uint64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t NewOpcode = MatchTable[CurrentIdx++]; + if (NewInsnID >= OutMIs.size()) + OutMIs.resize(NewInsnID + 1); + + OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), + State.MIs[OldInsnID]); OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); - DEBUG(dbgs() << "GIR_MutateOpcode(OutMIs[" << NewInsnID << "], MIs[" - << OldInsnID << "], " << NewOpcode << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "], " + << NewOpcode << ")\n"); break; } + case GIR_BuildMI: { - int64_t InsnID = *Command++; - int64_t Opcode = *Command++; - assert((size_t)InsnID == OutMIs.size() && - "Expected to store MIs in order"); - (void)InsnID; - OutMIs.push_back(BuildMI(*State.MIs[0]->getParent(), State.MIs[0], - State.MIs[0]->getDebugLoc(), TII.get(Opcode))); - DEBUG(dbgs() << "GIR_BuildMI(OutMIs[" << InsnID << "], " << Opcode - << ")\n"); + uint64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t Opcode = MatchTable[CurrentIdx++]; + if (NewInsnID >= OutMIs.size()) + OutMIs.resize(NewInsnID + 1); + + OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], + State.MIs[0]->getDebugLoc(), TII.get(Opcode)); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" + << NewInsnID << "], " << Opcode << ")\n"); break; } case GIR_Copy: { - int64_t NewInsnID = *Command++; - int64_t OldInsnID = *Command++; - int64_t OpIdx = *Command++; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); - DEBUG(dbgs() << "GIR_Copy(OutMIs[" << NewInsnID << "], MIs[" << OldInsnID - << "], " << OpIdx << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID + << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); break; } + + case GIR_CopyOrAddZeroReg: { + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t ZeroReg = MatchTable[CurrentIdx++]; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); + if (isOperandImmEqual(MO, 0, MRI)) + OutMIs[NewInsnID].addReg(ZeroReg); + else + OutMIs[NewInsnID].add(MO); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "], " + << OpIdx << ", " << ZeroReg << ")\n"); + break; + } + case GIR_CopySubReg: { - int64_t NewInsnID = *Command++; - int64_t OldInsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t SubRegIdx = *Command++; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t SubRegIdx = MatchTable[CurrentIdx++]; assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 0, SubRegIdx); - DEBUG(dbgs() << "GIR_CopySubReg(OutMIs[" << NewInsnID << "], MIs[" - << OldInsnID << "], " << OpIdx << ", " << SubRegIdx - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "], " + << OpIdx << ", " << SubRegIdx << ")\n"); break; } + case GIR_AddImplicitDef: { - int64_t InsnID = *Command++; - int64_t RegNum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RegNum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addDef(RegNum, RegState::Implicit); - DEBUG(dbgs() << "GIR_AddImplicitDef(OutMIs[" << InsnID << "], " << RegNum - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" + << InsnID << "], " << RegNum << ")\n"); break; } + case GIR_AddImplicitUse: { - int64_t InsnID = *Command++; - int64_t RegNum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RegNum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addUse(RegNum, RegState::Implicit); - DEBUG(dbgs() << "GIR_AddImplicitUse(OutMIs[" << InsnID << "], " << RegNum - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" + << InsnID << "], " << RegNum << ")\n"); break; } + case GIR_AddRegister: { - int64_t InsnID = *Command++; - int64_t RegNum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RegNum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addReg(RegNum); - DEBUG(dbgs() << "GIR_AddRegister(OutMIs[" << InsnID << "], " << RegNum - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" + << InsnID << "], " << RegNum << ")\n"); + break; + } + + case GIR_AddTempRegister: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t TempRegID = MatchTable[CurrentIdx++]; + uint64_t TempRegFlags = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" + << InsnID << "], TempRegisters[" << TempRegID + << "], " << TempRegFlags << ")\n"); break; } + case GIR_AddImm: { - int64_t InsnID = *Command++; - int64_t Imm = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Imm = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addImm(Imm); - DEBUG(dbgs() << "GIR_AddImm(OutMIs[" << InsnID << "], " << Imm << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID + << "], " << Imm << ")\n"); break; } + case GIR_ComplexRenderer: { - int64_t InsnID = *Command++; - int64_t RendererID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + for (const auto &RenderOpFn : State.Renderers[RendererID]) + RenderOpFn(OutMIs[InsnID]); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" + << InsnID << "], " << RendererID << ")\n"); + break; + } + case GIR_ComplexSubOperandRenderer: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + int64_t RenderOpID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); - State.Renderers[RendererID](OutMIs[InsnID]); - DEBUG(dbgs() << "GIR_ComplexRenderer(OutMIs[" << InsnID << "], " - << RendererID << ")\n"); + State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIR_ComplexSubOperandRenderer(OutMIs[" + << InsnID << "], " << RendererID << ", " + << RenderOpID << ")\n"); + break; + } + + case GIR_CopyConstantAsSImm: { + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { + OutMIs[NewInsnID].addImm( + State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); + } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) + OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); + else + llvm_unreachable("Expected Imm or CImm operand"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "])\n"); break; } case GIR_ConstrainOperandRC: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t RCEnum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RCEnum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, *TRI.getRegClass(RCEnum), TII, TRI, RBI); - DEBUG(dbgs() << "GIR_ConstrainOperandRC(OutMIs[" << InsnID << "], " - << OpIdx << ", " << RCEnum << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" + << InsnID << "], " << OpIdx << ", " << RCEnum + << ")\n"); break; } + case GIR_ConstrainSelectedInstOperands: { - int64_t InsnID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, RBI); - DEBUG(dbgs() << "GIR_ConstrainSelectedInstOperands(OutMIs[" << InsnID - << "])\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIR_ConstrainSelectedInstOperands(OutMIs[" + << InsnID << "])\n"); break; } + case GIR_MergeMemOperands: { - int64_t InsnID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); - for (const auto *FromMI : State.MIs) - for (const auto &MMO : FromMI->memoperands()) + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" + << InsnID << "]"); + int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; + while ((MergeInsnID = MatchTable[CurrentIdx++]) != + GIU_MergeMemOperands_EndOfList) { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << ", MIs[" << MergeInsnID << "]"); + for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) OutMIs[InsnID].addMemOperand(MMO); - DEBUG(dbgs() << "GIR_MergeMemOperands(OutMIs[" << InsnID << "])\n"); + } + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); break; } + case GIR_EraseFromParent: { - int64_t InsnID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; assert(State.MIs[InsnID] && "Attempted to erase an undefined instruction"); State.MIs[InsnID]->eraseFromParent(); - DEBUG(dbgs() << "GIR_EraseFromParent(MIs[" << InsnID << "])\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" + << InsnID << "])\n"); + break; + } + + case GIR_MakeTempReg: { + int64_t TempRegID = MatchTable[CurrentIdx++]; + int64_t TypeID = MatchTable[CurrentIdx++]; + + State.TempRegisters[TempRegID] = + MRI.createGenericVirtualRegister(MatcherInfo.TypeObjects[TypeID]); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": TempRegs[" << TempRegID + << "] = GIR_MakeTempReg(" << TypeID << ")\n"); + break; + } + + case GIR_Coverage: { + int64_t RuleID = MatchTable[CurrentIdx++]; + CoverageInfo.setCovered(RuleID); + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); break; } case GIR_Done: - DEBUG(dbgs() << "GIR_Done"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_Done"); return true; default: |