diff options
Diffstat (limited to 'contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp')
| -rw-r--r-- | contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp | 1048 |
1 files changed, 0 insertions, 1048 deletions
diff --git a/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp b/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp deleted file mode 100644 index 44175af7f9b6..000000000000 --- a/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp +++ /dev/null @@ -1,1048 +0,0 @@ -//===----------- PPCVSXSwapRemoval.cpp - Remove VSX LE Swaps -------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===---------------------------------------------------------------------===// -// -// This pass analyzes vector computations and removes unnecessary -// doubleword swaps (xxswapd instructions). This pass is performed -// only for little-endian VSX code generation. -// -// For this specific case, loads and stores of v4i32, v4f32, v2i64, -// and v2f64 vectors are inefficient. These are implemented using -// the lxvd2x and stxvd2x instructions, which invert the order of -// doublewords in a vector register. Thus code generation inserts -// an xxswapd after each such load, and prior to each such store. -// -// The extra xxswapd instructions reduce performance. The purpose -// of this pass is to reduce the number of xxswapd instructions -// required for correctness. -// -// The primary insight is that much code that operates on vectors -// does not care about the relative order of elements in a register, -// so long as the correct memory order is preserved. If we have a -// computation where all input values are provided by lxvd2x/xxswapd, -// all outputs are stored using xxswapd/lxvd2x, and all intermediate -// computations are lane-insensitive (independent of element order), -// then all the xxswapd instructions associated with the loads and -// stores may be removed without changing observable semantics. -// -// This pass uses standard equivalence class infrastructure to create -// maximal webs of computations fitting the above description. Each -// such web is then optimized by removing its unnecessary xxswapd -// instructions. -// -// There are some lane-sensitive operations for which we can still -// permit the optimization, provided we modify those operations -// accordingly. Such operations are identified as using "special -// handling" within this module. -// -//===---------------------------------------------------------------------===// - -#include "PPC.h" -#include "PPCInstrBuilder.h" -#include "PPCInstrInfo.h" -#include "PPCTargetMachine.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/EquivalenceClasses.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/raw_ostream.h" - -using namespace llvm; - -#define DEBUG_TYPE "ppc-vsx-swaps" - -namespace { - -// A PPCVSXSwapEntry is created for each machine instruction that -// is relevant to a vector computation. -struct PPCVSXSwapEntry { - // Pointer to the instruction. - MachineInstr *VSEMI; - - // Unique ID (position in the swap vector). - int VSEId; - - // Attributes of this node. - unsigned int IsLoad : 1; - unsigned int IsStore : 1; - unsigned int IsSwap : 1; - unsigned int MentionsPhysVR : 1; - unsigned int IsSwappable : 1; - unsigned int MentionsPartialVR : 1; - unsigned int SpecialHandling : 3; - unsigned int WebRejected : 1; - unsigned int WillRemove : 1; -}; - -enum SHValues { - SH_NONE = 0, - SH_EXTRACT, - SH_INSERT, - SH_NOSWAP_LD, - SH_NOSWAP_ST, - SH_SPLAT, - SH_XXPERMDI, - SH_COPYWIDEN -}; - -struct PPCVSXSwapRemoval : public MachineFunctionPass { - - static char ID; - const PPCInstrInfo *TII; - MachineFunction *MF; - MachineRegisterInfo *MRI; - - // Swap entries are allocated in a vector for better performance. - std::vector<PPCVSXSwapEntry> SwapVector; - - // A mapping is maintained between machine instructions and - // their swap entries. The key is the address of the MI. - DenseMap<MachineInstr*, int> SwapMap; - - // Equivalence classes are used to gather webs of related computation. - // Swap entries are represented by their VSEId fields. - EquivalenceClasses<int> *EC; - - PPCVSXSwapRemoval() : MachineFunctionPass(ID) { - initializePPCVSXSwapRemovalPass(*PassRegistry::getPassRegistry()); - } - -private: - // Initialize data structures. - void initialize(MachineFunction &MFParm); - - // Walk the machine instructions to gather vector usage information. - // Return true iff vector mentions are present. - bool gatherVectorInstructions(); - - // Add an entry to the swap vector and swap map. - int addSwapEntry(MachineInstr *MI, PPCVSXSwapEntry &SwapEntry); - - // Hunt backwards through COPY and SUBREG_TO_REG chains for a - // source register. VecIdx indicates the swap vector entry to - // mark as mentioning a physical register if the search leads - // to one. - unsigned lookThruCopyLike(unsigned SrcReg, unsigned VecIdx); - - // Generate equivalence classes for related computations (webs). - void formWebs(); - - // Analyze webs and determine those that cannot be optimized. - void recordUnoptimizableWebs(); - - // Record which swap instructions can be safely removed. - void markSwapsForRemoval(); - - // Remove swaps and update other instructions requiring special - // handling. Return true iff any changes are made. - bool removeSwaps(); - - // Insert a swap instruction from SrcReg to DstReg at the given - // InsertPoint. - void insertSwap(MachineInstr *MI, MachineBasicBlock::iterator InsertPoint, - unsigned DstReg, unsigned SrcReg); - - // Update instructions requiring special handling. - void handleSpecialSwappables(int EntryIdx); - - // Dump a description of the entries in the swap vector. - void dumpSwapVector(); - - // Return true iff the given register is in the given class. - bool isRegInClass(unsigned Reg, const TargetRegisterClass *RC) { - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return RC->hasSubClassEq(MRI->getRegClass(Reg)); - return RC->contains(Reg); - } - - // Return true iff the given register is a full vector register. - bool isVecReg(unsigned Reg) { - return (isRegInClass(Reg, &PPC::VSRCRegClass) || - isRegInClass(Reg, &PPC::VRRCRegClass)); - } - - // Return true iff the given register is a partial vector register. - bool isScalarVecReg(unsigned Reg) { - return (isRegInClass(Reg, &PPC::VSFRCRegClass) || - isRegInClass(Reg, &PPC::VSSRCRegClass)); - } - - // Return true iff the given register mentions all or part of a - // vector register. Also sets Partial to true if the mention - // is for just the floating-point register overlap of the register. - bool isAnyVecReg(unsigned Reg, bool &Partial) { - if (isScalarVecReg(Reg)) - Partial = true; - return isScalarVecReg(Reg) || isVecReg(Reg); - } - -public: - // Main entry point for this pass. - bool runOnMachineFunction(MachineFunction &MF) override { - if (skipFunction(MF.getFunction())) - return false; - - // If we don't have VSX on the subtarget, don't do anything. - // Also, on Power 9 the load and store ops preserve element order and so - // the swaps are not required. - const PPCSubtarget &STI = MF.getSubtarget<PPCSubtarget>(); - if (!STI.hasVSX() || !STI.needsSwapsForVSXMemOps()) - return false; - - bool Changed = false; - initialize(MF); - - if (gatherVectorInstructions()) { - formWebs(); - recordUnoptimizableWebs(); - markSwapsForRemoval(); - Changed = removeSwaps(); - } - - // FIXME: See the allocation of EC in initialize(). - delete EC; - return Changed; - } -}; - -// Initialize data structures for this pass. In particular, clear the -// swap vector and allocate the equivalence class mapping before -// processing each function. -void PPCVSXSwapRemoval::initialize(MachineFunction &MFParm) { - MF = &MFParm; - MRI = &MF->getRegInfo(); - TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo(); - - // An initial vector size of 256 appears to work well in practice. - // Small/medium functions with vector content tend not to incur a - // reallocation at this size. Three of the vector tests in - // projects/test-suite reallocate, which seems like a reasonable rate. - const int InitialVectorSize(256); - SwapVector.clear(); - SwapVector.reserve(InitialVectorSize); - - // FIXME: Currently we allocate EC each time because we don't have - // access to the set representation on which to call clear(). Should - // consider adding a clear() method to the EquivalenceClasses class. - EC = new EquivalenceClasses<int>; -} - -// Create an entry in the swap vector for each instruction that mentions -// a full vector register, recording various characteristics of the -// instructions there. -bool PPCVSXSwapRemoval::gatherVectorInstructions() { - bool RelevantFunction = false; - - for (MachineBasicBlock &MBB : *MF) { - for (MachineInstr &MI : MBB) { - - if (MI.isDebugInstr()) - continue; - - bool RelevantInstr = false; - bool Partial = false; - - for (const MachineOperand &MO : MI.operands()) { - if (!MO.isReg()) - continue; - unsigned Reg = MO.getReg(); - if (isAnyVecReg(Reg, Partial)) { - RelevantInstr = true; - break; - } - } - - if (!RelevantInstr) - continue; - - RelevantFunction = true; - - // Create a SwapEntry initialized to zeros, then fill in the - // instruction and ID fields before pushing it to the back - // of the swap vector. - PPCVSXSwapEntry SwapEntry{}; - int VecIdx = addSwapEntry(&MI, SwapEntry); - - switch(MI.getOpcode()) { - default: - // Unless noted otherwise, an instruction is considered - // safe for the optimization. There are a large number of - // such true-SIMD instructions (all vector math, logical, - // select, compare, etc.). However, if the instruction - // mentions a partial vector register and does not have - // special handling defined, it is not swappable. - if (Partial) - SwapVector[VecIdx].MentionsPartialVR = 1; - else - SwapVector[VecIdx].IsSwappable = 1; - break; - case PPC::XXPERMDI: { - // This is a swap if it is of the form XXPERMDI t, s, s, 2. - // Unfortunately, MachineCSE ignores COPY and SUBREG_TO_REG, so we - // can also see XXPERMDI t, SUBREG_TO_REG(s), SUBREG_TO_REG(s), 2, - // for example. We have to look through chains of COPY and - // SUBREG_TO_REG to find the real source value for comparison. - // If the real source value is a physical register, then mark the - // XXPERMDI as mentioning a physical register. - int immed = MI.getOperand(3).getImm(); - if (immed == 2) { - unsigned trueReg1 = lookThruCopyLike(MI.getOperand(1).getReg(), - VecIdx); - unsigned trueReg2 = lookThruCopyLike(MI.getOperand(2).getReg(), - VecIdx); - if (trueReg1 == trueReg2) - SwapVector[VecIdx].IsSwap = 1; - else { - // We can still handle these if the two registers are not - // identical, by adjusting the form of the XXPERMDI. - SwapVector[VecIdx].IsSwappable = 1; - SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI; - } - // This is a doubleword splat if it is of the form - // XXPERMDI t, s, s, 0 or XXPERMDI t, s, s, 3. As above we - // must look through chains of copy-likes to find the source - // register. We turn off the marking for mention of a physical - // register, because splatting it is safe; the optimization - // will not swap the value in the physical register. Whether - // or not the two input registers are identical, we can handle - // these by adjusting the form of the XXPERMDI. - } else if (immed == 0 || immed == 3) { - - SwapVector[VecIdx].IsSwappable = 1; - SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI; - - unsigned trueReg1 = lookThruCopyLike(MI.getOperand(1).getReg(), - VecIdx); - unsigned trueReg2 = lookThruCopyLike(MI.getOperand(2).getReg(), - VecIdx); - if (trueReg1 == trueReg2) - SwapVector[VecIdx].MentionsPhysVR = 0; - - } else { - // We can still handle these by adjusting the form of the XXPERMDI. - SwapVector[VecIdx].IsSwappable = 1; - SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI; - } - break; - } - case PPC::LVX: - // Non-permuting loads are currently unsafe. We can use special - // handling for this in the future. By not marking these as - // IsSwap, we ensure computations containing them will be rejected - // for now. - SwapVector[VecIdx].IsLoad = 1; - break; - case PPC::LXVD2X: - case PPC::LXVW4X: - // Permuting loads are marked as both load and swap, and are - // safe for optimization. - SwapVector[VecIdx].IsLoad = 1; - SwapVector[VecIdx].IsSwap = 1; - break; - case PPC::LXSDX: - case PPC::LXSSPX: - case PPC::XFLOADf64: - case PPC::XFLOADf32: - // A load of a floating-point value into the high-order half of - // a vector register is safe, provided that we introduce a swap - // following the load, which will be done by the SUBREG_TO_REG - // support. So just mark these as safe. - SwapVector[VecIdx].IsLoad = 1; - SwapVector[VecIdx].IsSwappable = 1; - break; - case PPC::STVX: - // Non-permuting stores are currently unsafe. We can use special - // handling for this in the future. By not marking these as - // IsSwap, we ensure computations containing them will be rejected - // for now. - SwapVector[VecIdx].IsStore = 1; - break; - case PPC::STXVD2X: - case PPC::STXVW4X: - // Permuting stores are marked as both store and swap, and are - // safe for optimization. - SwapVector[VecIdx].IsStore = 1; - SwapVector[VecIdx].IsSwap = 1; - break; - case PPC::COPY: - // These are fine provided they are moving between full vector - // register classes. - if (isVecReg(MI.getOperand(0).getReg()) && - isVecReg(MI.getOperand(1).getReg())) - SwapVector[VecIdx].IsSwappable = 1; - // If we have a copy from one scalar floating-point register - // to another, we can accept this even if it is a physical - // register. The only way this gets involved is if it feeds - // a SUBREG_TO_REG, which is handled by introducing a swap. - else if (isScalarVecReg(MI.getOperand(0).getReg()) && - isScalarVecReg(MI.getOperand(1).getReg())) - SwapVector[VecIdx].IsSwappable = 1; - break; - case PPC::SUBREG_TO_REG: { - // These are fine provided they are moving between full vector - // register classes. If they are moving from a scalar - // floating-point class to a vector class, we can handle those - // as well, provided we introduce a swap. It is generally the - // case that we will introduce fewer swaps than we remove, but - // (FIXME) a cost model could be used. However, introduced - // swaps could potentially be CSEd, so this is not trivial. - if (isVecReg(MI.getOperand(0).getReg()) && - isVecReg(MI.getOperand(2).getReg())) - SwapVector[VecIdx].IsSwappable = 1; - else if (isVecReg(MI.getOperand(0).getReg()) && - isScalarVecReg(MI.getOperand(2).getReg())) { - SwapVector[VecIdx].IsSwappable = 1; - SwapVector[VecIdx].SpecialHandling = SHValues::SH_COPYWIDEN; - } - break; - } - case PPC::VSPLTB: - case PPC::VSPLTH: - case PPC::VSPLTW: - case PPC::XXSPLTW: - // Splats are lane-sensitive, but we can use special handling - // to adjust the source lane for the splat. - SwapVector[VecIdx].IsSwappable = 1; - SwapVector[VecIdx].SpecialHandling = SHValues::SH_SPLAT; - break; - // The presence of the following lane-sensitive operations in a - // web will kill the optimization, at least for now. For these - // we do nothing, causing the optimization to fail. - // FIXME: Some of these could be permitted with special handling, - // and will be phased in as time permits. - // FIXME: There is no simple and maintainable way to express a set - // of opcodes having a common attribute in TableGen. Should this - // change, this is a prime candidate to use such a mechanism. - case PPC::INLINEASM: - case PPC::INLINEASM_BR: - case PPC::EXTRACT_SUBREG: - case PPC::INSERT_SUBREG: - case PPC::COPY_TO_REGCLASS: - case PPC::LVEBX: - case PPC::LVEHX: - case PPC::LVEWX: - case PPC::LVSL: - case PPC::LVSR: - case PPC::LVXL: - case PPC::STVEBX: - case PPC::STVEHX: - case PPC::STVEWX: - case PPC::STVXL: - // We can handle STXSDX and STXSSPX similarly to LXSDX and LXSSPX, - // by adding special handling for narrowing copies as well as - // widening ones. However, I've experimented with this, and in - // practice we currently do not appear to use STXSDX fed by - // a narrowing copy from a full vector register. Since I can't - // generate any useful test cases, I've left this alone for now. - case PPC::STXSDX: - case PPC::STXSSPX: - case PPC::VCIPHER: - case PPC::VCIPHERLAST: - case PPC::VMRGHB: - case PPC::VMRGHH: - case PPC::VMRGHW: - case PPC::VMRGLB: - case PPC::VMRGLH: - case PPC::VMRGLW: - case PPC::VMULESB: - case PPC::VMULESH: - case PPC::VMULESW: - case PPC::VMULEUB: - case PPC::VMULEUH: - case PPC::VMULEUW: - case PPC::VMULOSB: - case PPC::VMULOSH: - case PPC::VMULOSW: - case PPC::VMULOUB: - case PPC::VMULOUH: - case PPC::VMULOUW: - case PPC::VNCIPHER: - case PPC::VNCIPHERLAST: - case PPC::VPERM: - case PPC::VPERMXOR: - case PPC::VPKPX: - case PPC::VPKSHSS: - case PPC::VPKSHUS: - case PPC::VPKSDSS: - case PPC::VPKSDUS: - case PPC::VPKSWSS: - case PPC::VPKSWUS: - case PPC::VPKUDUM: - case PPC::VPKUDUS: - case PPC::VPKUHUM: - case PPC::VPKUHUS: - case PPC::VPKUWUM: - case PPC::VPKUWUS: - case PPC::VPMSUMB: - case PPC::VPMSUMD: - case PPC::VPMSUMH: - case PPC::VPMSUMW: - case PPC::VRLB: - case PPC::VRLD: - case PPC::VRLH: - case PPC::VRLW: - case PPC::VSBOX: - case PPC::VSHASIGMAD: - case PPC::VSHASIGMAW: - case PPC::VSL: - case PPC::VSLDOI: - case PPC::VSLO: - case PPC::VSR: - case PPC::VSRO: - case PPC::VSUM2SWS: - case PPC::VSUM4SBS: - case PPC::VSUM4SHS: - case PPC::VSUM4UBS: - case PPC::VSUMSWS: - case PPC::VUPKHPX: - case PPC::VUPKHSB: - case PPC::VUPKHSH: - case PPC::VUPKHSW: - case PPC::VUPKLPX: - case PPC::VUPKLSB: - case PPC::VUPKLSH: - case PPC::VUPKLSW: - case PPC::XXMRGHW: - case PPC::XXMRGLW: - // XXSLDWI could be replaced by a general permute with one of three - // permute control vectors (for shift values 1, 2, 3). However, - // VPERM has a more restrictive register class. - case PPC::XXSLDWI: - case PPC::XSCVDPSPN: - case PPC::XSCVSPDPN: - break; - } - } - } - - if (RelevantFunction) { - LLVM_DEBUG(dbgs() << "Swap vector when first built\n\n"); - LLVM_DEBUG(dumpSwapVector()); - } - - return RelevantFunction; -} - -// Add an entry to the swap vector and swap map, and make a -// singleton equivalence class for the entry. -int PPCVSXSwapRemoval::addSwapEntry(MachineInstr *MI, - PPCVSXSwapEntry& SwapEntry) { - SwapEntry.VSEMI = MI; - SwapEntry.VSEId = SwapVector.size(); - SwapVector.push_back(SwapEntry); - EC->insert(SwapEntry.VSEId); - SwapMap[MI] = SwapEntry.VSEId; - return SwapEntry.VSEId; -} - -// This is used to find the "true" source register for an -// XXPERMDI instruction, since MachineCSE does not handle the -// "copy-like" operations (Copy and SubregToReg). Returns -// the original SrcReg unless it is the target of a copy-like -// operation, in which case we chain backwards through all -// such operations to the ultimate source register. If a -// physical register is encountered, we stop the search and -// flag the swap entry indicated by VecIdx (the original -// XXPERMDI) as mentioning a physical register. -unsigned PPCVSXSwapRemoval::lookThruCopyLike(unsigned SrcReg, - unsigned VecIdx) { - MachineInstr *MI = MRI->getVRegDef(SrcReg); - if (!MI->isCopyLike()) - return SrcReg; - - unsigned CopySrcReg; - if (MI->isCopy()) - CopySrcReg = MI->getOperand(1).getReg(); - else { - assert(MI->isSubregToReg() && "bad opcode for lookThruCopyLike"); - CopySrcReg = MI->getOperand(2).getReg(); - } - - if (!TargetRegisterInfo::isVirtualRegister(CopySrcReg)) { - if (!isScalarVecReg(CopySrcReg)) - SwapVector[VecIdx].MentionsPhysVR = 1; - return CopySrcReg; - } - - return lookThruCopyLike(CopySrcReg, VecIdx); -} - -// Generate equivalence classes for related computations (webs) by -// def-use relationships of virtual registers. Mention of a physical -// register terminates the generation of equivalence classes as this -// indicates a use of a parameter, definition of a return value, use -// of a value returned from a call, or definition of a parameter to a -// call. Computations with physical register mentions are flagged -// as such so their containing webs will not be optimized. -void PPCVSXSwapRemoval::formWebs() { - - LLVM_DEBUG(dbgs() << "\n*** Forming webs for swap removal ***\n\n"); - - for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { - - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - - LLVM_DEBUG(dbgs() << "\n" << SwapVector[EntryIdx].VSEId << " "); - LLVM_DEBUG(MI->dump()); - - // It's sufficient to walk vector uses and join them to their unique - // definitions. In addition, check full vector register operands - // for physical regs. We exclude partial-vector register operands - // because we can handle them if copied to a full vector. - for (const MachineOperand &MO : MI->operands()) { - if (!MO.isReg()) - continue; - - unsigned Reg = MO.getReg(); - if (!isVecReg(Reg) && !isScalarVecReg(Reg)) - continue; - - if (!TargetRegisterInfo::isVirtualRegister(Reg)) { - if (!(MI->isCopy() && isScalarVecReg(Reg))) - SwapVector[EntryIdx].MentionsPhysVR = 1; - continue; - } - - if (!MO.isUse()) - continue; - - MachineInstr* DefMI = MRI->getVRegDef(Reg); - assert(SwapMap.find(DefMI) != SwapMap.end() && - "Inconsistency: def of vector reg not found in swap map!"); - int DefIdx = SwapMap[DefMI]; - (void)EC->unionSets(SwapVector[DefIdx].VSEId, - SwapVector[EntryIdx].VSEId); - - LLVM_DEBUG(dbgs() << format("Unioning %d with %d\n", - SwapVector[DefIdx].VSEId, - SwapVector[EntryIdx].VSEId)); - LLVM_DEBUG(dbgs() << " Def: "); - LLVM_DEBUG(DefMI->dump()); - } - } -} - -// Walk the swap vector entries looking for conditions that prevent their -// containing computations from being optimized. When such conditions are -// found, mark the representative of the computation's equivalence class -// as rejected. -void PPCVSXSwapRemoval::recordUnoptimizableWebs() { - - LLVM_DEBUG(dbgs() << "\n*** Rejecting webs for swap removal ***\n\n"); - - for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { - int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); - - // If representative is already rejected, don't waste further time. - if (SwapVector[Repr].WebRejected) - continue; - - // Reject webs containing mentions of physical or partial registers, or - // containing operations that we don't know how to handle in a lane- - // permuted region. - if (SwapVector[EntryIdx].MentionsPhysVR || - SwapVector[EntryIdx].MentionsPartialVR || - !(SwapVector[EntryIdx].IsSwappable || SwapVector[EntryIdx].IsSwap)) { - - SwapVector[Repr].WebRejected = 1; - - LLVM_DEBUG( - dbgs() << format("Web %d rejected for physreg, partial reg, or not " - "swap[pable]\n", - Repr)); - LLVM_DEBUG(dbgs() << " in " << EntryIdx << ": "); - LLVM_DEBUG(SwapVector[EntryIdx].VSEMI->dump()); - LLVM_DEBUG(dbgs() << "\n"); - } - - // Reject webs than contain swapping loads that feed something other - // than a swap instruction. - else if (SwapVector[EntryIdx].IsLoad && SwapVector[EntryIdx].IsSwap) { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - unsigned DefReg = MI->getOperand(0).getReg(); - - // We skip debug instructions in the analysis. (Note that debug - // location information is still maintained by this optimization - // because it remains on the LXVD2X and STXVD2X instructions after - // the XXPERMDIs are removed.) - for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) { - int UseIdx = SwapMap[&UseMI]; - - if (!SwapVector[UseIdx].IsSwap || SwapVector[UseIdx].IsLoad || - SwapVector[UseIdx].IsStore) { - - SwapVector[Repr].WebRejected = 1; - - LLVM_DEBUG(dbgs() << format( - "Web %d rejected for load not feeding swap\n", Repr)); - LLVM_DEBUG(dbgs() << " def " << EntryIdx << ": "); - LLVM_DEBUG(MI->dump()); - LLVM_DEBUG(dbgs() << " use " << UseIdx << ": "); - LLVM_DEBUG(UseMI.dump()); - LLVM_DEBUG(dbgs() << "\n"); - } - } - - // Reject webs that contain swapping stores that are fed by something - // other than a swap instruction. - } else if (SwapVector[EntryIdx].IsStore && SwapVector[EntryIdx].IsSwap) { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - unsigned UseReg = MI->getOperand(0).getReg(); - MachineInstr *DefMI = MRI->getVRegDef(UseReg); - unsigned DefReg = DefMI->getOperand(0).getReg(); - int DefIdx = SwapMap[DefMI]; - - if (!SwapVector[DefIdx].IsSwap || SwapVector[DefIdx].IsLoad || - SwapVector[DefIdx].IsStore) { - - SwapVector[Repr].WebRejected = 1; - - LLVM_DEBUG(dbgs() << format( - "Web %d rejected for store not fed by swap\n", Repr)); - LLVM_DEBUG(dbgs() << " def " << DefIdx << ": "); - LLVM_DEBUG(DefMI->dump()); - LLVM_DEBUG(dbgs() << " use " << EntryIdx << ": "); - LLVM_DEBUG(MI->dump()); - LLVM_DEBUG(dbgs() << "\n"); - } - - // Ensure all uses of the register defined by DefMI feed store - // instructions - for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) { - int UseIdx = SwapMap[&UseMI]; - - if (SwapVector[UseIdx].VSEMI->getOpcode() != MI->getOpcode()) { - SwapVector[Repr].WebRejected = 1; - - LLVM_DEBUG( - dbgs() << format( - "Web %d rejected for swap not feeding only stores\n", Repr)); - LLVM_DEBUG(dbgs() << " def " - << " : "); - LLVM_DEBUG(DefMI->dump()); - LLVM_DEBUG(dbgs() << " use " << UseIdx << ": "); - LLVM_DEBUG(SwapVector[UseIdx].VSEMI->dump()); - LLVM_DEBUG(dbgs() << "\n"); - } - } - } - } - - LLVM_DEBUG(dbgs() << "Swap vector after web analysis:\n\n"); - LLVM_DEBUG(dumpSwapVector()); -} - -// Walk the swap vector entries looking for swaps fed by permuting loads -// and swaps that feed permuting stores. If the containing computation -// has not been marked rejected, mark each such swap for removal. -// (Removal is delayed in case optimization has disturbed the pattern, -// such that multiple loads feed the same swap, etc.) -void PPCVSXSwapRemoval::markSwapsForRemoval() { - - LLVM_DEBUG(dbgs() << "\n*** Marking swaps for removal ***\n\n"); - - for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { - - if (SwapVector[EntryIdx].IsLoad && SwapVector[EntryIdx].IsSwap) { - int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); - - if (!SwapVector[Repr].WebRejected) { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - unsigned DefReg = MI->getOperand(0).getReg(); - - for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) { - int UseIdx = SwapMap[&UseMI]; - SwapVector[UseIdx].WillRemove = 1; - - LLVM_DEBUG(dbgs() << "Marking swap fed by load for removal: "); - LLVM_DEBUG(UseMI.dump()); - } - } - - } else if (SwapVector[EntryIdx].IsStore && SwapVector[EntryIdx].IsSwap) { - int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); - - if (!SwapVector[Repr].WebRejected) { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - unsigned UseReg = MI->getOperand(0).getReg(); - MachineInstr *DefMI = MRI->getVRegDef(UseReg); - int DefIdx = SwapMap[DefMI]; - SwapVector[DefIdx].WillRemove = 1; - - LLVM_DEBUG(dbgs() << "Marking swap feeding store for removal: "); - LLVM_DEBUG(DefMI->dump()); - } - - } else if (SwapVector[EntryIdx].IsSwappable && - SwapVector[EntryIdx].SpecialHandling != 0) { - int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); - - if (!SwapVector[Repr].WebRejected) - handleSpecialSwappables(EntryIdx); - } - } -} - -// Create an xxswapd instruction and insert it prior to the given point. -// MI is used to determine basic block and debug loc information. -// FIXME: When inserting a swap, we should check whether SrcReg is -// defined by another swap: SrcReg = XXPERMDI Reg, Reg, 2; If so, -// then instead we should generate a copy from Reg to DstReg. -void PPCVSXSwapRemoval::insertSwap(MachineInstr *MI, - MachineBasicBlock::iterator InsertPoint, - unsigned DstReg, unsigned SrcReg) { - BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(), - TII->get(PPC::XXPERMDI), DstReg) - .addReg(SrcReg) - .addReg(SrcReg) - .addImm(2); -} - -// The identified swap entry requires special handling to allow its -// containing computation to be optimized. Perform that handling -// here. -// FIXME: Additional opportunities will be phased in with subsequent -// patches. -void PPCVSXSwapRemoval::handleSpecialSwappables(int EntryIdx) { - switch (SwapVector[EntryIdx].SpecialHandling) { - - default: - llvm_unreachable("Unexpected special handling type"); - - // For splats based on an index into a vector, add N/2 modulo N - // to the index, where N is the number of vector elements. - case SHValues::SH_SPLAT: { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - unsigned NElts; - - LLVM_DEBUG(dbgs() << "Changing splat: "); - LLVM_DEBUG(MI->dump()); - - switch (MI->getOpcode()) { - default: - llvm_unreachable("Unexpected splat opcode"); - case PPC::VSPLTB: NElts = 16; break; - case PPC::VSPLTH: NElts = 8; break; - case PPC::VSPLTW: - case PPC::XXSPLTW: NElts = 4; break; - } - - unsigned EltNo; - if (MI->getOpcode() == PPC::XXSPLTW) - EltNo = MI->getOperand(2).getImm(); - else - EltNo = MI->getOperand(1).getImm(); - - EltNo = (EltNo + NElts / 2) % NElts; - if (MI->getOpcode() == PPC::XXSPLTW) - MI->getOperand(2).setImm(EltNo); - else - MI->getOperand(1).setImm(EltNo); - - LLVM_DEBUG(dbgs() << " Into: "); - LLVM_DEBUG(MI->dump()); - break; - } - - // For an XXPERMDI that isn't handled otherwise, we need to - // reverse the order of the operands. If the selector operand - // has a value of 0 or 3, we need to change it to 3 or 0, - // respectively. Otherwise we should leave it alone. (This - // is equivalent to reversing the two bits of the selector - // operand and complementing the result.) - case SHValues::SH_XXPERMDI: { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - - LLVM_DEBUG(dbgs() << "Changing XXPERMDI: "); - LLVM_DEBUG(MI->dump()); - - unsigned Selector = MI->getOperand(3).getImm(); - if (Selector == 0 || Selector == 3) - Selector = 3 - Selector; - MI->getOperand(3).setImm(Selector); - - unsigned Reg1 = MI->getOperand(1).getReg(); - unsigned Reg2 = MI->getOperand(2).getReg(); - MI->getOperand(1).setReg(Reg2); - MI->getOperand(2).setReg(Reg1); - - // We also need to swap kill flag associated with the register. - bool IsKill1 = MI->getOperand(1).isKill(); - bool IsKill2 = MI->getOperand(2).isKill(); - MI->getOperand(1).setIsKill(IsKill2); - MI->getOperand(2).setIsKill(IsKill1); - - LLVM_DEBUG(dbgs() << " Into: "); - LLVM_DEBUG(MI->dump()); - break; - } - - // For a copy from a scalar floating-point register to a vector - // register, removing swaps will leave the copied value in the - // wrong lane. Insert a swap following the copy to fix this. - case SHValues::SH_COPYWIDEN: { - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - - LLVM_DEBUG(dbgs() << "Changing SUBREG_TO_REG: "); - LLVM_DEBUG(MI->dump()); - - unsigned DstReg = MI->getOperand(0).getReg(); - const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); - unsigned NewVReg = MRI->createVirtualRegister(DstRC); - - MI->getOperand(0).setReg(NewVReg); - LLVM_DEBUG(dbgs() << " Into: "); - LLVM_DEBUG(MI->dump()); - - auto InsertPoint = ++MachineBasicBlock::iterator(MI); - - // Note that an XXPERMDI requires a VSRC, so if the SUBREG_TO_REG - // is copying to a VRRC, we need to be careful to avoid a register - // assignment problem. In this case we must copy from VRRC to VSRC - // prior to the swap, and from VSRC to VRRC following the swap. - // Coalescing will usually remove all this mess. - if (DstRC == &PPC::VRRCRegClass) { - unsigned VSRCTmp1 = MRI->createVirtualRegister(&PPC::VSRCRegClass); - unsigned VSRCTmp2 = MRI->createVirtualRegister(&PPC::VSRCRegClass); - - BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(), - TII->get(PPC::COPY), VSRCTmp1) - .addReg(NewVReg); - LLVM_DEBUG(std::prev(InsertPoint)->dump()); - - insertSwap(MI, InsertPoint, VSRCTmp2, VSRCTmp1); - LLVM_DEBUG(std::prev(InsertPoint)->dump()); - - BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(), - TII->get(PPC::COPY), DstReg) - .addReg(VSRCTmp2); - LLVM_DEBUG(std::prev(InsertPoint)->dump()); - - } else { - insertSwap(MI, InsertPoint, DstReg, NewVReg); - LLVM_DEBUG(std::prev(InsertPoint)->dump()); - } - break; - } - } -} - -// Walk the swap vector and replace each entry marked for removal with -// a copy operation. -bool PPCVSXSwapRemoval::removeSwaps() { - - LLVM_DEBUG(dbgs() << "\n*** Removing swaps ***\n\n"); - - bool Changed = false; - - for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { - if (SwapVector[EntryIdx].WillRemove) { - Changed = true; - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - MachineBasicBlock *MBB = MI->getParent(); - BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(TargetOpcode::COPY), - MI->getOperand(0).getReg()) - .add(MI->getOperand(1)); - - LLVM_DEBUG(dbgs() << format("Replaced %d with copy: ", - SwapVector[EntryIdx].VSEId)); - LLVM_DEBUG(MI->dump()); - - MI->eraseFromParent(); - } - } - - return Changed; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -// For debug purposes, dump the contents of the swap vector. -LLVM_DUMP_METHOD void PPCVSXSwapRemoval::dumpSwapVector() { - - for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { - - MachineInstr *MI = SwapVector[EntryIdx].VSEMI; - int ID = SwapVector[EntryIdx].VSEId; - - dbgs() << format("%6d", ID); - dbgs() << format("%6d", EC->getLeaderValue(ID)); - dbgs() << format(" %bb.%3d", MI->getParent()->getNumber()); - dbgs() << format(" %14s ", TII->getName(MI->getOpcode()).str().c_str()); - - if (SwapVector[EntryIdx].IsLoad) - dbgs() << "load "; - if (SwapVector[EntryIdx].IsStore) - dbgs() << "store "; - if (SwapVector[EntryIdx].IsSwap) - dbgs() << "swap "; - if (SwapVector[EntryIdx].MentionsPhysVR) - dbgs() << "physreg "; - if (SwapVector[EntryIdx].MentionsPartialVR) - dbgs() << "partialreg "; - - if (SwapVector[EntryIdx].IsSwappable) { - dbgs() << "swappable "; - switch(SwapVector[EntryIdx].SpecialHandling) { - default: - dbgs() << "special:**unknown**"; - break; - case SH_NONE: - break; - case SH_EXTRACT: - dbgs() << "special:extract "; - break; - case SH_INSERT: - dbgs() << "special:insert "; - break; - case SH_NOSWAP_LD: - dbgs() << "special:load "; - break; - case SH_NOSWAP_ST: - dbgs() << "special:store "; - break; - case SH_SPLAT: - dbgs() << "special:splat "; - break; - case SH_XXPERMDI: - dbgs() << "special:xxpermdi "; - break; - case SH_COPYWIDEN: - dbgs() << "special:copywiden "; - break; - } - } - - if (SwapVector[EntryIdx].WebRejected) - dbgs() << "rejected "; - if (SwapVector[EntryIdx].WillRemove) - dbgs() << "remove "; - - dbgs() << "\n"; - - // For no-asserts builds. - (void)MI; - (void)ID; - } - - dbgs() << "\n"; -} -#endif - -} // end default namespace - -INITIALIZE_PASS_BEGIN(PPCVSXSwapRemoval, DEBUG_TYPE, - "PowerPC VSX Swap Removal", false, false) -INITIALIZE_PASS_END(PPCVSXSwapRemoval, DEBUG_TYPE, - "PowerPC VSX Swap Removal", false, false) - -char PPCVSXSwapRemoval::ID = 0; -FunctionPass* -llvm::createPPCVSXSwapRemovalPass() { return new PPCVSXSwapRemoval(); } |