diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Target/Mips/MipsExpandPseudo.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/Target/Mips/MipsExpandPseudo.cpp | 913 |
1 files changed, 913 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/contrib/llvm-project/llvm/lib/Target/Mips/MipsExpandPseudo.cpp new file mode 100644 index 000000000000..b1abf4a33717 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/Target/Mips/MipsExpandPseudo.cpp @@ -0,0 +1,913 @@ +//===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains a pass that expands pseudo instructions into target +// instructions to allow proper scheduling, if-conversion, and other late +// optimizations. This pass should be run after register allocation but before +// the post-regalloc scheduling pass. +// +// This is currently only used for expanding atomic pseudos after register +// allocation. We do this to avoid the fast register allocator introducing +// spills between ll and sc. These stores cause some MIPS implementations to +// abort the atomic RMW sequence. +// +//===----------------------------------------------------------------------===// + +#include "Mips.h" +#include "MipsInstrInfo.h" +#include "MipsSubtarget.h" +#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" + +using namespace llvm; + +#define DEBUG_TYPE "mips-pseudo" + +namespace { + class MipsExpandPseudo : public MachineFunctionPass { + public: + static char ID; + MipsExpandPseudo() : MachineFunctionPass(ID) {} + + const MipsInstrInfo *TII; + const MipsSubtarget *STI; + + bool runOnMachineFunction(MachineFunction &Fn) override; + + MachineFunctionProperties getRequiredProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoVRegs); + } + + StringRef getPassName() const override { + return "Mips pseudo instruction expansion pass"; + } + + private: + bool expandAtomicCmpSwap(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + + bool expandAtomicBinOp(MachineBasicBlock &BB, + MachineBasicBlock::iterator I, + MachineBasicBlock::iterator &NMBBI, unsigned Size); + bool expandAtomicBinOpSubword(MachineBasicBlock &BB, + MachineBasicBlock::iterator I, + MachineBasicBlock::iterator &NMBBI); + + bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NMBB); + bool expandMBB(MachineBasicBlock &MBB); + }; + char MipsExpandPseudo::ID = 0; +} + +bool MipsExpandPseudo::expandAtomicCmpSwapSubword( + MachineBasicBlock &BB, MachineBasicBlock::iterator I, + MachineBasicBlock::iterator &NMBBI) { + + MachineFunction *MF = BB.getParent(); + + const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); + DebugLoc DL = I->getDebugLoc(); + unsigned LL, SC; + + unsigned ZERO = Mips::ZERO; + unsigned BNE = Mips::BNE; + unsigned BEQ = Mips::BEQ; + unsigned SEOp = + I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH; + + if (STI->inMicroMipsMode()) { + LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; + SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; + BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; + BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; + } else { + LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) + : (ArePtrs64bit ? Mips::LL64 : Mips::LL); + SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) + : (ArePtrs64bit ? Mips::SC64 : Mips::SC); + } + + Register Dest = I->getOperand(0).getReg(); + Register Ptr = I->getOperand(1).getReg(); + Register Mask = I->getOperand(2).getReg(); + Register ShiftCmpVal = I->getOperand(3).getReg(); + Register Mask2 = I->getOperand(4).getReg(); + Register ShiftNewVal = I->getOperand(5).getReg(); + Register ShiftAmnt = I->getOperand(6).getReg(); + Register Scratch = I->getOperand(7).getReg(); + Register Scratch2 = I->getOperand(8).getReg(); + + // insert new blocks after the current block + const BasicBlock *LLVM_BB = BB.getBasicBlock(); + MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineFunction::iterator It = ++BB.getIterator(); + MF->insert(It, loop1MBB); + MF->insert(It, loop2MBB); + MF->insert(It, sinkMBB); + MF->insert(It, exitMBB); + + // Transfer the remainder of BB and its successor edges to exitMBB. + exitMBB->splice(exitMBB->begin(), &BB, + std::next(MachineBasicBlock::iterator(I)), BB.end()); + exitMBB->transferSuccessorsAndUpdatePHIs(&BB); + + // thisMBB: + // ... + // fallthrough --> loop1MBB + BB.addSuccessor(loop1MBB, BranchProbability::getOne()); + loop1MBB->addSuccessor(sinkMBB); + loop1MBB->addSuccessor(loop2MBB); + loop1MBB->normalizeSuccProbs(); + loop2MBB->addSuccessor(loop1MBB); + loop2MBB->addSuccessor(sinkMBB); + loop2MBB->normalizeSuccProbs(); + sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); + + // loop1MBB: + // ll dest, 0(ptr) + // and Mask', dest, Mask + // bne Mask', ShiftCmpVal, exitMBB + BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0); + BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2) + .addReg(Scratch) + .addReg(Mask); + BuildMI(loop1MBB, DL, TII->get(BNE)) + .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB); + + // loop2MBB: + // and dest, dest, mask2 + // or dest, dest, ShiftNewVal + // sc dest, dest, 0(ptr) + // beq dest, $0, loop1MBB + BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch) + .addReg(Scratch, RegState::Kill) + .addReg(Mask2); + BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch) + .addReg(Scratch, RegState::Kill) + .addReg(ShiftNewVal); + BuildMI(loop2MBB, DL, TII->get(SC), Scratch) + .addReg(Scratch, RegState::Kill) + .addReg(Ptr) + .addImm(0); + BuildMI(loop2MBB, DL, TII->get(BEQ)) + .addReg(Scratch, RegState::Kill) + .addReg(ZERO) + .addMBB(loop1MBB); + + // sinkMBB: + // srl srlres, Mask', shiftamt + // sign_extend dest,srlres + BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) + .addReg(Scratch2) + .addReg(ShiftAmnt); + if (STI->hasMips32r2()) { + BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); + } else { + const unsigned ShiftImm = + I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24; + BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) + .addReg(Dest, RegState::Kill) + .addImm(ShiftImm); + BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) + .addReg(Dest, RegState::Kill) + .addImm(ShiftImm); + } + + LivePhysRegs LiveRegs; + computeAndAddLiveIns(LiveRegs, *loop1MBB); + computeAndAddLiveIns(LiveRegs, *loop2MBB); + computeAndAddLiveIns(LiveRegs, *sinkMBB); + computeAndAddLiveIns(LiveRegs, *exitMBB); + + NMBBI = BB.end(); + I->eraseFromParent(); + return true; +} + +bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB, + MachineBasicBlock::iterator I, + MachineBasicBlock::iterator &NMBBI) { + + const unsigned Size = + I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8; + MachineFunction *MF = BB.getParent(); + + const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); + DebugLoc DL = I->getDebugLoc(); + + unsigned LL, SC, ZERO, BNE, BEQ, MOVE; + + if (Size == 4) { + if (STI->inMicroMipsMode()) { + LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; + SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; + BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; + BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; + } else { + LL = STI->hasMips32r6() + ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) + : (ArePtrs64bit ? Mips::LL64 : Mips::LL); + SC = STI->hasMips32r6() + ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) + : (ArePtrs64bit ? Mips::SC64 : Mips::SC); + BNE = Mips::BNE; + BEQ = Mips::BEQ; + } + + ZERO = Mips::ZERO; + MOVE = Mips::OR; + } else { + LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; + SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; + ZERO = Mips::ZERO_64; + BNE = Mips::BNE64; + BEQ = Mips::BEQ64; + MOVE = Mips::OR64; + } + + Register Dest = I->getOperand(0).getReg(); + Register Ptr = I->getOperand(1).getReg(); + Register OldVal = I->getOperand(2).getReg(); + Register NewVal = I->getOperand(3).getReg(); + Register Scratch = I->getOperand(4).getReg(); + + // insert new blocks after the current block + const BasicBlock *LLVM_BB = BB.getBasicBlock(); + MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineFunction::iterator It = ++BB.getIterator(); + MF->insert(It, loop1MBB); + MF->insert(It, loop2MBB); + MF->insert(It, exitMBB); + + // Transfer the remainder of BB and its successor edges to exitMBB. + exitMBB->splice(exitMBB->begin(), &BB, + std::next(MachineBasicBlock::iterator(I)), BB.end()); + exitMBB->transferSuccessorsAndUpdatePHIs(&BB); + + // thisMBB: + // ... + // fallthrough --> loop1MBB + BB.addSuccessor(loop1MBB, BranchProbability::getOne()); + loop1MBB->addSuccessor(exitMBB); + loop1MBB->addSuccessor(loop2MBB); + loop1MBB->normalizeSuccProbs(); + loop2MBB->addSuccessor(loop1MBB); + loop2MBB->addSuccessor(exitMBB); + loop2MBB->normalizeSuccProbs(); + + // loop1MBB: + // ll dest, 0(ptr) + // bne dest, oldval, exitMBB + BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0); + BuildMI(loop1MBB, DL, TII->get(BNE)) + .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB); + + // loop2MBB: + // move scratch, NewVal + // sc Scratch, Scratch, 0(ptr) + // beq Scratch, $0, loop1MBB + BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO); + BuildMI(loop2MBB, DL, TII->get(SC), Scratch) + .addReg(Scratch).addReg(Ptr).addImm(0); + BuildMI(loop2MBB, DL, TII->get(BEQ)) + .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB); + + LivePhysRegs LiveRegs; + computeAndAddLiveIns(LiveRegs, *loop1MBB); + computeAndAddLiveIns(LiveRegs, *loop2MBB); + computeAndAddLiveIns(LiveRegs, *exitMBB); + + NMBBI = BB.end(); + I->eraseFromParent(); + return true; +} + +bool MipsExpandPseudo::expandAtomicBinOpSubword( + MachineBasicBlock &BB, MachineBasicBlock::iterator I, + MachineBasicBlock::iterator &NMBBI) { + + MachineFunction *MF = BB.getParent(); + + const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); + DebugLoc DL = I->getDebugLoc(); + + unsigned LL, SC, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ; + unsigned BEQ = Mips::BEQ; + unsigned SEOp = Mips::SEH; + + if (STI->inMicroMipsMode()) { + LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; + SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; + BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; + SLT = Mips::SLT_MM; + SLTu = Mips::SLTu_MM; + OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM; + MOVN = Mips::MOVN_I_MM; + MOVZ = Mips::MOVZ_I_MM; + SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ; + SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ; + } else { + LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) + : (ArePtrs64bit ? Mips::LL64 : Mips::LL); + SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) + : (ArePtrs64bit ? Mips::SC64 : Mips::SC); + SLT = Mips::SLT; + SLTu = Mips::SLTu; + OR = Mips::OR; + MOVN = Mips::MOVN_I_I; + MOVZ = Mips::MOVZ_I_I; + SELNEZ = Mips::SELNEZ; + SELEQZ = Mips::SELEQZ; + } + + bool IsSwap = false; + bool IsNand = false; + bool IsMin = false; + bool IsMax = false; + bool IsUnsigned = false; + + unsigned Opcode = 0; + switch (I->getOpcode()) { + case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: + IsNand = true; + break; + case Mips::ATOMIC_SWAP_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_SWAP_I16_POSTRA: + IsSwap = true; + break; + case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: + Opcode = Mips::ADDu; + break; + case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: + Opcode = Mips::SUBu; + break; + case Mips::ATOMIC_LOAD_AND_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_AND_I16_POSTRA: + Opcode = Mips::AND; + break; + case Mips::ATOMIC_LOAD_OR_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_OR_I16_POSTRA: + Opcode = Mips::OR; + break; + case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: + SEOp = Mips::SEB; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: + Opcode = Mips::XOR; + break; + case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA: + case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA: + IsUnsigned = true; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_MIN_I8_POSTRA: + case Mips::ATOMIC_LOAD_MIN_I16_POSTRA: + IsMin = true; + break; + case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA: + case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA: + IsUnsigned = true; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_MAX_I8_POSTRA: + case Mips::ATOMIC_LOAD_MAX_I16_POSTRA: + IsMax = true; + break; + default: + llvm_unreachable("Unknown subword atomic pseudo for expansion!"); + } + + Register Dest = I->getOperand(0).getReg(); + Register Ptr = I->getOperand(1).getReg(); + Register Incr = I->getOperand(2).getReg(); + Register Mask = I->getOperand(3).getReg(); + Register Mask2 = I->getOperand(4).getReg(); + Register ShiftAmnt = I->getOperand(5).getReg(); + Register OldVal = I->getOperand(6).getReg(); + Register BinOpRes = I->getOperand(7).getReg(); + Register StoreVal = I->getOperand(8).getReg(); + + const BasicBlock *LLVM_BB = BB.getBasicBlock(); + MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineFunction::iterator It = ++BB.getIterator(); + MF->insert(It, loopMBB); + MF->insert(It, sinkMBB); + MF->insert(It, exitMBB); + + exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); + exitMBB->transferSuccessorsAndUpdatePHIs(&BB); + + BB.addSuccessor(loopMBB, BranchProbability::getOne()); + loopMBB->addSuccessor(sinkMBB); + loopMBB->addSuccessor(loopMBB); + loopMBB->normalizeSuccProbs(); + + BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); + if (IsNand) { + // and andres, oldval, incr2 + // nor binopres, $0, andres + // and newval, binopres, mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(OldVal) + .addReg(Incr); + BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes) + .addReg(Mips::ZERO) + .addReg(BinOpRes); + BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(BinOpRes) + .addReg(Mask); + } else if (IsMin || IsMax) { + + assert(I->getNumOperands() == 10 && + "Atomics min|max|umin|umax use an additional register"); + Register Scratch4 = I->getOperand(9).getReg(); + + unsigned SLTScratch4 = IsUnsigned ? SLTu : SLT; + unsigned SELIncr = IsMax ? SELNEZ : SELEQZ; + unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ; + unsigned MOVIncr = IsMax ? MOVN : MOVZ; + + // For little endian we need to clear uninterested bits. + if (STI->isLittle()) { + // and OldVal, OldVal, Mask + // and Incr, Incr, Mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal) + .addReg(OldVal) + .addReg(Mask); + BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr).addReg(Incr).addReg(Mask); + } + + // unsigned: sltu Scratch4, oldVal, Incr + // signed: slt Scratch4, oldVal, Incr + BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4) + .addReg(OldVal) + .addReg(Incr); + + if (STI->hasMips64r6() || STI->hasMips32r6()) { + // max: seleqz BinOpRes, OldVal, Scratch4 + // selnez Scratch4, Incr, Scratch4 + // or BinOpRes, BinOpRes, Scratch4 + // min: selnqz BinOpRes, OldVal, Scratch4 + // seleqz Scratch4, Incr, Scratch4 + // or BinOpRes, BinOpRes, Scratch4 + BuildMI(loopMBB, DL, TII->get(SELOldVal), BinOpRes) + .addReg(OldVal) + .addReg(Scratch4); + BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch4) + .addReg(Incr) + .addReg(Scratch4); + BuildMI(loopMBB, DL, TII->get(OR), BinOpRes) + .addReg(BinOpRes) + .addReg(Scratch4); + } else { + // max: move BinOpRes, OldVal + // movn BinOpRes, Incr, Scratch4, BinOpRes + // min: move BinOpRes, OldVal + // movz BinOpRes, Incr, Scratch4, BinOpRes + BuildMI(loopMBB, DL, TII->get(OR), BinOpRes) + .addReg(OldVal) + .addReg(Mips::ZERO); + BuildMI(loopMBB, DL, TII->get(MOVIncr), BinOpRes) + .addReg(Incr) + .addReg(Scratch4) + .addReg(BinOpRes); + } + + // and BinOpRes, BinOpRes, Mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(BinOpRes) + .addReg(Mask); + + } else if (!IsSwap) { + // <binop> binopres, oldval, incr2 + // and newval, binopres, mask + BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes) + .addReg(OldVal) + .addReg(Incr); + BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(BinOpRes) + .addReg(Mask); + } else { // atomic.swap + // and newval, incr2, mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(Incr) + .addReg(Mask); + } + + // and StoreVal, OlddVal, Mask2 + // or StoreVal, StoreVal, BinOpRes + // StoreVal<tied1> = sc StoreVal, 0(Ptr) + // beq StoreVal, zero, loopMBB + BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) + .addReg(OldVal).addReg(Mask2); + BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) + .addReg(StoreVal).addReg(BinOpRes); + BuildMI(loopMBB, DL, TII->get(SC), StoreVal) + .addReg(StoreVal).addReg(Ptr).addImm(0); + BuildMI(loopMBB, DL, TII->get(BEQ)) + .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB); + + // sinkMBB: + // and maskedoldval1,oldval,mask + // srl srlres,maskedoldval1,shiftamt + // sign_extend dest,srlres + + sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); + + BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest) + .addReg(OldVal).addReg(Mask); + BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) + .addReg(Dest).addReg(ShiftAmnt); + + if (STI->hasMips32r2()) { + BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); + } else { + const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24; + BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) + .addReg(Dest, RegState::Kill) + .addImm(ShiftImm); + BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) + .addReg(Dest, RegState::Kill) + .addImm(ShiftImm); + } + + LivePhysRegs LiveRegs; + computeAndAddLiveIns(LiveRegs, *loopMBB); + computeAndAddLiveIns(LiveRegs, *sinkMBB); + computeAndAddLiveIns(LiveRegs, *exitMBB); + + NMBBI = BB.end(); + I->eraseFromParent(); + + return true; +} + +bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, + MachineBasicBlock::iterator I, + MachineBasicBlock::iterator &NMBBI, + unsigned Size) { + MachineFunction *MF = BB.getParent(); + + const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); + DebugLoc DL = I->getDebugLoc(); + + unsigned LL, SC, ZERO, BEQ, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ; + + if (Size == 4) { + if (STI->inMicroMipsMode()) { + LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; + SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; + BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; + SLT = Mips::SLT_MM; + SLTu = Mips::SLTu_MM; + OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM; + MOVN = Mips::MOVN_I_MM; + MOVZ = Mips::MOVZ_I_MM; + SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ; + SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ; + } else { + LL = STI->hasMips32r6() + ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) + : (ArePtrs64bit ? Mips::LL64 : Mips::LL); + SC = STI->hasMips32r6() + ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) + : (ArePtrs64bit ? Mips::SC64 : Mips::SC); + BEQ = Mips::BEQ; + SLT = Mips::SLT; + SLTu = Mips::SLTu; + OR = Mips::OR; + MOVN = Mips::MOVN_I_I; + MOVZ = Mips::MOVZ_I_I; + SELNEZ = Mips::SELNEZ; + SELEQZ = Mips::SELEQZ; + } + + ZERO = Mips::ZERO; + } else { + LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; + SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; + ZERO = Mips::ZERO_64; + BEQ = Mips::BEQ64; + SLT = Mips::SLT64; + SLTu = Mips::SLTu64; + OR = Mips::OR64; + MOVN = Mips::MOVN_I64_I64; + MOVZ = Mips::MOVZ_I64_I64; + SELNEZ = Mips::SELNEZ64; + SELEQZ = Mips::SELEQZ64; + } + + Register OldVal = I->getOperand(0).getReg(); + Register Ptr = I->getOperand(1).getReg(); + Register Incr = I->getOperand(2).getReg(); + Register Scratch = I->getOperand(3).getReg(); + + unsigned Opcode = 0; + unsigned AND = 0; + unsigned NOR = 0; + + bool IsOr = false; + bool IsNand = false; + bool IsMin = false; + bool IsMax = false; + bool IsUnsigned = false; + + switch (I->getOpcode()) { + case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: + Opcode = Mips::ADDu; + break; + case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: + Opcode = Mips::SUBu; + break; + case Mips::ATOMIC_LOAD_AND_I32_POSTRA: + Opcode = Mips::AND; + break; + case Mips::ATOMIC_LOAD_OR_I32_POSTRA: + Opcode = Mips::OR; + break; + case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: + Opcode = Mips::XOR; + break; + case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: + IsNand = true; + AND = Mips::AND; + NOR = Mips::NOR; + break; + case Mips::ATOMIC_SWAP_I32_POSTRA: + IsOr = true; + break; + case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: + Opcode = Mips::DADDu; + break; + case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: + Opcode = Mips::DSUBu; + break; + case Mips::ATOMIC_LOAD_AND_I64_POSTRA: + Opcode = Mips::AND64; + break; + case Mips::ATOMIC_LOAD_OR_I64_POSTRA: + Opcode = Mips::OR64; + break; + case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: + Opcode = Mips::XOR64; + break; + case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: + IsNand = true; + AND = Mips::AND64; + NOR = Mips::NOR64; + break; + case Mips::ATOMIC_SWAP_I64_POSTRA: + IsOr = true; + break; + case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA: + case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA: + IsUnsigned = true; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_MIN_I32_POSTRA: + case Mips::ATOMIC_LOAD_MIN_I64_POSTRA: + IsMin = true; + break; + case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA: + case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA: + IsUnsigned = true; + LLVM_FALLTHROUGH; + case Mips::ATOMIC_LOAD_MAX_I32_POSTRA: + case Mips::ATOMIC_LOAD_MAX_I64_POSTRA: + IsMax = true; + break; + default: + llvm_unreachable("Unknown pseudo atomic!"); + } + + const BasicBlock *LLVM_BB = BB.getBasicBlock(); + MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineFunction::iterator It = ++BB.getIterator(); + MF->insert(It, loopMBB); + MF->insert(It, exitMBB); + + exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); + exitMBB->transferSuccessorsAndUpdatePHIs(&BB); + + BB.addSuccessor(loopMBB, BranchProbability::getOne()); + loopMBB->addSuccessor(exitMBB); + loopMBB->addSuccessor(loopMBB); + loopMBB->normalizeSuccProbs(); + + BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); + assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!"); + assert((OldVal != Incr) && "Clobbered the wrong reg!"); + if (IsMin || IsMax) { + + assert(I->getNumOperands() == 5 && + "Atomics min|max|umin|umax use an additional register"); + Register Scratch2 = I->getOperand(4).getReg(); + + // On Mips64 result of slt is GPR32. + Register Scratch2_32 = + (Size == 8) ? STI->getRegisterInfo()->getSubReg(Scratch2, Mips::sub_32) + : Scratch2; + + unsigned SLTScratch2 = IsUnsigned ? SLTu : SLT; + unsigned SELIncr = IsMax ? SELNEZ : SELEQZ; + unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ; + unsigned MOVIncr = IsMax ? MOVN : MOVZ; + + // unsigned: sltu Scratch2, oldVal, Incr + // signed: slt Scratch2, oldVal, Incr + BuildMI(loopMBB, DL, TII->get(SLTScratch2), Scratch2_32) + .addReg(OldVal) + .addReg(Incr); + + if (STI->hasMips64r6() || STI->hasMips32r6()) { + // max: seleqz Scratch, OldVal, Scratch2 + // selnez Scratch2, Incr, Scratch2 + // or Scratch, Scratch, Scratch2 + // min: selnez Scratch, OldVal, Scratch2 + // seleqz Scratch2, Incr, Scratch2 + // or Scratch, Scratch, Scratch2 + BuildMI(loopMBB, DL, TII->get(SELOldVal), Scratch) + .addReg(OldVal) + .addReg(Scratch2); + BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch2) + .addReg(Incr) + .addReg(Scratch2); + BuildMI(loopMBB, DL, TII->get(OR), Scratch) + .addReg(Scratch) + .addReg(Scratch2); + } else { + // max: move Scratch, OldVal + // movn Scratch, Incr, Scratch2, Scratch + // min: move Scratch, OldVal + // movz Scratch, Incr, Scratch2, Scratch + BuildMI(loopMBB, DL, TII->get(OR), Scratch) + .addReg(OldVal) + .addReg(ZERO); + BuildMI(loopMBB, DL, TII->get(MOVIncr), Scratch) + .addReg(Incr) + .addReg(Scratch2) + .addReg(Scratch); + } + + } else if (Opcode) { + BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr); + } else if (IsNand) { + assert(AND && NOR && + "Unknown nand instruction for atomic pseudo expansion"); + BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr); + BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch); + } else { + assert(IsOr && OR && "Unknown instruction for atomic pseudo expansion!"); + (void)IsOr; + BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); + } + + BuildMI(loopMBB, DL, TII->get(SC), Scratch) + .addReg(Scratch) + .addReg(Ptr) + .addImm(0); + BuildMI(loopMBB, DL, TII->get(BEQ)) + .addReg(Scratch) + .addReg(ZERO) + .addMBB(loopMBB); + + NMBBI = BB.end(); + I->eraseFromParent(); + + LivePhysRegs LiveRegs; + computeAndAddLiveIns(LiveRegs, *loopMBB); + computeAndAddLiveIns(LiveRegs, *exitMBB); + + return true; +} + +bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NMBB) { + + bool Modified = false; + + switch (MBBI->getOpcode()) { + case Mips::ATOMIC_CMP_SWAP_I32_POSTRA: + case Mips::ATOMIC_CMP_SWAP_I64_POSTRA: + return expandAtomicCmpSwap(MBB, MBBI, NMBB); + case Mips::ATOMIC_CMP_SWAP_I8_POSTRA: + case Mips::ATOMIC_CMP_SWAP_I16_POSTRA: + return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB); + case Mips::ATOMIC_SWAP_I8_POSTRA: + case Mips::ATOMIC_SWAP_I16_POSTRA: + case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: + case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: + case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: + case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: + case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: + case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: + case Mips::ATOMIC_LOAD_AND_I8_POSTRA: + case Mips::ATOMIC_LOAD_AND_I16_POSTRA: + case Mips::ATOMIC_LOAD_OR_I8_POSTRA: + case Mips::ATOMIC_LOAD_OR_I16_POSTRA: + case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: + case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: + case Mips::ATOMIC_LOAD_MIN_I8_POSTRA: + case Mips::ATOMIC_LOAD_MIN_I16_POSTRA: + case Mips::ATOMIC_LOAD_MAX_I8_POSTRA: + case Mips::ATOMIC_LOAD_MAX_I16_POSTRA: + case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA: + case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA: + case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA: + case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA: + return expandAtomicBinOpSubword(MBB, MBBI, NMBB); + case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: + case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: + case Mips::ATOMIC_LOAD_AND_I32_POSTRA: + case Mips::ATOMIC_LOAD_OR_I32_POSTRA: + case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: + case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: + case Mips::ATOMIC_SWAP_I32_POSTRA: + case Mips::ATOMIC_LOAD_MIN_I32_POSTRA: + case Mips::ATOMIC_LOAD_MAX_I32_POSTRA: + case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA: + case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA: + return expandAtomicBinOp(MBB, MBBI, NMBB, 4); + case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: + case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: + case Mips::ATOMIC_LOAD_AND_I64_POSTRA: + case Mips::ATOMIC_LOAD_OR_I64_POSTRA: + case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: + case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: + case Mips::ATOMIC_SWAP_I64_POSTRA: + case Mips::ATOMIC_LOAD_MIN_I64_POSTRA: + case Mips::ATOMIC_LOAD_MAX_I64_POSTRA: + case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA: + case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA: + return expandAtomicBinOp(MBB, MBBI, NMBB, 8); + default: + return Modified; + } +} + +bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) { + bool Modified = false; + + MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); + while (MBBI != E) { + MachineBasicBlock::iterator NMBBI = std::next(MBBI); + Modified |= expandMI(MBB, MBBI, NMBBI); + MBBI = NMBBI; + } + + return Modified; +} + +bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) { + STI = &static_cast<const MipsSubtarget &>(MF.getSubtarget()); + TII = STI->getInstrInfo(); + + bool Modified = false; + for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E; + ++MFI) + Modified |= expandMBB(*MFI); + + if (Modified) + MF.RenumberBlocks(); + + return Modified; +} + +/// createMipsExpandPseudoPass - returns an instance of the pseudo instruction +/// expansion pass. +FunctionPass *llvm::createMipsExpandPseudoPass() { + return new MipsExpandPseudo(); +} |