diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2017-01-02 19:17:04 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2017-01-02 19:17:04 +0000 |
| commit | b915e9e0fc85ba6f398b3fab0db6a81a8913af94 (patch) | |
| tree | 98b8f811c7aff2547cab8642daf372d6c59502fb /lib/Target/AMDGPU/SIFrameLowering.cpp | |
| parent | 6421cca32f69ac849537a3cff78c352195e99f1b (diff) | |
Notes
Diffstat (limited to 'lib/Target/AMDGPU/SIFrameLowering.cpp')
| -rw-r--r-- | lib/Target/AMDGPU/SIFrameLowering.cpp | 401 |
1 files changed, 231 insertions, 170 deletions
diff --git a/lib/Target/AMDGPU/SIFrameLowering.cpp b/lib/Target/AMDGPU/SIFrameLowering.cpp index 03b11f0fd38d..d0a69eafc58e 100644 --- a/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -21,20 +21,168 @@ using namespace llvm; -static bool hasOnlySGPRSpills(const SIMachineFunctionInfo *FuncInfo, - const MachineFrameInfo *FrameInfo) { - return FuncInfo->hasSpilledSGPRs() && - (!FuncInfo->hasSpilledVGPRs() && !FuncInfo->hasNonSpillStackObjects()); -} - -static ArrayRef<MCPhysReg> getAllSGPR128() { +static ArrayRef<MCPhysReg> getAllSGPR128(const MachineFunction &MF, + const SIRegisterInfo *TRI) { return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(), - AMDGPU::SGPR_128RegClass.getNumRegs()); + TRI->getMaxNumSGPRs(MF) / 4); } -static ArrayRef<MCPhysReg> getAllSGPRs() { +static ArrayRef<MCPhysReg> getAllSGPRs(const MachineFunction &MF, + const SIRegisterInfo *TRI) { return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), - AMDGPU::SGPR_32RegClass.getNumRegs()); + TRI->getMaxNumSGPRs(MF)); +} + +void SIFrameLowering::emitFlatScratchInit(const SIInstrInfo *TII, + const SIRegisterInfo* TRI, + MachineFunction &MF, + MachineBasicBlock &MBB) const { + // We don't need this if we only have spills since there is no user facing + // scratch. + + // TODO: If we know we don't have flat instructions earlier, we can omit + // this from the input registers. + // + // TODO: We only need to know if we access scratch space through a flat + // pointer. Because we only detect if flat instructions are used at all, + // this will be used more often than necessary on VI. + + // Debug location must be unknown since the first debug location is used to + // determine the end of the prologue. + DebugLoc DL; + MachineBasicBlock::iterator I = MBB.begin(); + + unsigned FlatScratchInitReg + = TRI->getPreloadedValue(MF, SIRegisterInfo::FLAT_SCRATCH_INIT); + + MachineRegisterInfo &MRI = MF.getRegInfo(); + MRI.addLiveIn(FlatScratchInitReg); + MBB.addLiveIn(FlatScratchInitReg); + + // Copy the size in bytes. + unsigned FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1); + BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO) + .addReg(FlatScrInitHi, RegState::Kill); + + unsigned FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0); + + const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); + unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg(); + + // Add wave offset in bytes to private base offset. + // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init. + BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo) + .addReg(FlatScrInitLo) + .addReg(ScratchWaveOffsetReg); + + // Convert offset to 256-byte units. + BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI) + .addReg(FlatScrInitLo, RegState::Kill) + .addImm(8); +} + +unsigned SIFrameLowering::getReservedPrivateSegmentBufferReg( + const SISubtarget &ST, + const SIInstrInfo *TII, + const SIRegisterInfo *TRI, + SIMachineFunctionInfo *MFI, + MachineFunction &MF) const { + + // We need to insert initialization of the scratch resource descriptor. + unsigned ScratchRsrcReg = MFI->getScratchRSrcReg(); + if (ScratchRsrcReg == AMDGPU::NoRegister) + return AMDGPU::NoRegister; + + if (ST.hasSGPRInitBug() || + ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF)) + return ScratchRsrcReg; + + // We reserved the last registers for this. Shift it down to the end of those + // which were actually used. + // + // FIXME: It might be safer to use a pseudoregister before replacement. + + // FIXME: We should be able to eliminate unused input registers. We only + // cannot do this for the resources required for scratch access. For now we + // skip over user SGPRs and may leave unused holes. + + // We find the resource first because it has an alignment requirement. + + MachineRegisterInfo &MRI = MF.getRegInfo(); + + unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4; + ArrayRef<MCPhysReg> AllSGPR128s = getAllSGPR128(MF, TRI); + AllSGPR128s = AllSGPR128s.slice(std::min(static_cast<unsigned>(AllSGPR128s.size()), NumPreloaded)); + + // Skip the last 2 elements because the last one is reserved for VCC, and + // this is the 2nd to last element already. + for (MCPhysReg Reg : AllSGPR128s) { + // Pick the first unallocated one. Make sure we don't clobber the other + // reserved input we needed. + if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg)) { + //assert(MRI.isAllocatable(Reg)); + MRI.replaceRegWith(ScratchRsrcReg, Reg); + MFI->setScratchRSrcReg(Reg); + return Reg; + } + } + + return ScratchRsrcReg; +} + +unsigned SIFrameLowering::getReservedPrivateSegmentWaveByteOffsetReg( + const SISubtarget &ST, + const SIInstrInfo *TII, + const SIRegisterInfo *TRI, + SIMachineFunctionInfo *MFI, + MachineFunction &MF) const { + unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg(); + if (ST.hasSGPRInitBug() || + ScratchWaveOffsetReg != TRI->reservedPrivateSegmentWaveByteOffsetReg(MF)) + return ScratchWaveOffsetReg; + + unsigned ScratchRsrcReg = MFI->getScratchRSrcReg(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + + unsigned NumPreloaded = MFI->getNumPreloadedSGPRs(); + + ArrayRef<MCPhysReg> AllSGPRs = getAllSGPRs(MF, TRI); + if (NumPreloaded > AllSGPRs.size()) + return ScratchWaveOffsetReg; + + AllSGPRs = AllSGPRs.slice(NumPreloaded); + + // We need to drop register from the end of the list that we cannot use + // for the scratch wave offset. + // + 2 s102 and s103 do not exist on VI. + // + 2 for vcc + // + 2 for xnack_mask + // + 2 for flat_scratch + // + 4 for registers reserved for scratch resource register + // + 1 for register reserved for scratch wave offset. (By exluding this + // register from the list to consider, it means that when this + // register is being used for the scratch wave offset and there + // are no other free SGPRs, then the value will stay in this register. + // ---- + // 13 + if (AllSGPRs.size() < 13) + return ScratchWaveOffsetReg; + + for (MCPhysReg Reg : AllSGPRs.drop_back(13)) { + // Pick the first unallocated SGPR. Be careful not to pick an alias of the + // scratch descriptor, since we haven’t added its uses yet. + if (!MRI.isPhysRegUsed(Reg)) { + if (!MRI.isAllocatable(Reg) || + TRI->isSubRegisterEq(ScratchRsrcReg, Reg)) + continue; + + MRI.replaceRegWith(ScratchWaveOffsetReg, Reg); + MFI->setScratchWaveOffsetReg(Reg); + return Reg; + } + } + + return ScratchWaveOffsetReg; } void SIFrameLowering::emitPrologue(MachineFunction &MF, @@ -45,9 +193,6 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, if (ST.debuggerEmitPrologue()) emitDebuggerPrologue(MF, MBB); - if (!MF.getFrameInfo()->hasStackObjects()) - return; - assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); @@ -57,186 +202,111 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, // // FIXME: We should be cleaning up these unused SGPR spill frame indices // somewhere. - if (hasOnlySGPRSpills(MFI, MF.getFrameInfo())) - return; const SIInstrInfo *TII = ST.getInstrInfo(); const SIRegisterInfo *TRI = &TII->getRegisterInfo(); MachineRegisterInfo &MRI = MF.getRegInfo(); - MachineBasicBlock::iterator I = MBB.begin(); - - // We need to insert initialization of the scratch resource descriptor. - unsigned ScratchRsrcReg = MFI->getScratchRSrcReg(); - assert(ScratchRsrcReg != AMDGPU::NoRegister); - unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg(); - assert(ScratchWaveOffsetReg != AMDGPU::NoRegister); - - unsigned PreloadedScratchWaveOffsetReg = TRI->getPreloadedValue( - MF, SIRegisterInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET); + unsigned ScratchRsrcReg + = getReservedPrivateSegmentBufferReg(ST, TII, TRI, MFI, MF); + unsigned ScratchWaveOffsetReg + = getReservedPrivateSegmentWaveByteOffsetReg(ST, TII, TRI, MFI, MF); - unsigned PreloadedPrivateBufferReg = AMDGPU::NoRegister; - if (ST.isAmdHsaOS()) { - PreloadedPrivateBufferReg = TRI->getPreloadedValue( - MF, SIRegisterInfo::PRIVATE_SEGMENT_BUFFER); + if (ScratchRsrcReg == AMDGPU::NoRegister) { + assert(ScratchWaveOffsetReg == AMDGPU::NoRegister); + return; } - if (MFI->hasFlatScratchInit()) { - // We don't need this if we only have spills since there is no user facing - // scratch. - - // TODO: If we know we don't have flat instructions earlier, we can omit - // this from the input registers. - // - // TODO: We only need to know if we access scratch space through a flat - // pointer. Because we only detect if flat instructions are used at all, - // this will be used more often than necessary on VI. - - // Debug location must be unknown since the first debug location is used to - // determine the end of the prologue. - DebugLoc DL; - - unsigned FlatScratchInitReg - = TRI->getPreloadedValue(MF, SIRegisterInfo::FLAT_SCRATCH_INIT); + assert(!TRI->isSubRegister(ScratchRsrcReg, ScratchWaveOffsetReg)); - MRI.addLiveIn(FlatScratchInitReg); - MBB.addLiveIn(FlatScratchInitReg); + // We need to do the replacement of the private segment buffer and wave offset + // register even if there are no stack objects. There could be stores to undef + // or a constant without an associated object. - // Copy the size in bytes. - unsigned FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1); - BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::FLAT_SCR_LO) - .addReg(FlatScrInitHi, RegState::Kill); + // FIXME: We still have implicit uses on SGPR spill instructions in case they + // need to spill to vector memory. It's likely that will not happen, but at + // this point it appears we need the setup. This part of the prolog should be + // emitted after frame indices are eliminated. - unsigned FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0); + if (MF.getFrameInfo().hasStackObjects() && MFI->hasFlatScratchInit()) + emitFlatScratchInit(TII, TRI, MF, MBB); - // Add wave offset in bytes to private base offset. - // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init. - BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo) - .addReg(FlatScrInitLo) - .addReg(ScratchWaveOffsetReg); + // We need to insert initialization of the scratch resource descriptor. + unsigned PreloadedScratchWaveOffsetReg = TRI->getPreloadedValue( + MF, SIRegisterInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET); - // Convert offset to 256-byte units. - BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI) - .addReg(FlatScrInitLo, RegState::Kill) - .addImm(8); - } - // If we reserved the original input registers, we don't need to copy to the - // reserved registers. - if (ScratchRsrcReg == PreloadedPrivateBufferReg) { - // We should always reserve these 5 registers at the same time. - assert(ScratchWaveOffsetReg == PreloadedScratchWaveOffsetReg && - "scratch wave offset and private segment buffer inconsistent"); - return; + unsigned PreloadedPrivateBufferReg = AMDGPU::NoRegister; + if (ST.isAmdCodeObjectV2()) { + PreloadedPrivateBufferReg = TRI->getPreloadedValue( + MF, SIRegisterInfo::PRIVATE_SEGMENT_BUFFER); } + bool OffsetRegUsed = !MRI.use_empty(ScratchWaveOffsetReg); + bool ResourceRegUsed = !MRI.use_empty(ScratchRsrcReg); // We added live-ins during argument lowering, but since they were not used // they were deleted. We're adding the uses now, so add them back. - MRI.addLiveIn(PreloadedScratchWaveOffsetReg); - MBB.addLiveIn(PreloadedScratchWaveOffsetReg); + if (OffsetRegUsed) { + assert(PreloadedScratchWaveOffsetReg != AMDGPU::NoRegister && + "scratch wave offset input is required"); + MRI.addLiveIn(PreloadedScratchWaveOffsetReg); + MBB.addLiveIn(PreloadedScratchWaveOffsetReg); + } - if (ST.isAmdHsaOS()) { + if (ResourceRegUsed && PreloadedPrivateBufferReg != AMDGPU::NoRegister) { + assert(ST.isAmdCodeObjectV2()); MRI.addLiveIn(PreloadedPrivateBufferReg); MBB.addLiveIn(PreloadedPrivateBufferReg); } - if (!ST.hasSGPRInitBug()) { - // We reserved the last registers for this. Shift it down to the end of those - // which were actually used. - // - // FIXME: It might be safer to use a pseudoregister before replacement. + // Make the register selected live throughout the function. + for (MachineBasicBlock &OtherBB : MF) { + if (&OtherBB == &MBB) + continue; - // FIXME: We should be able to eliminate unused input registers. We only - // cannot do this for the resources required for scratch access. For now we - // skip over user SGPRs and may leave unused holes. + if (OffsetRegUsed) + OtherBB.addLiveIn(ScratchWaveOffsetReg); - // We find the resource first because it has an alignment requirement. - if (ScratchRsrcReg == TRI->reservedPrivateSegmentBufferReg(MF)) { - MachineRegisterInfo &MRI = MF.getRegInfo(); + if (ResourceRegUsed) + OtherBB.addLiveIn(ScratchRsrcReg); + } - unsigned NumPreloaded = MFI->getNumPreloadedSGPRs() / 4; - // Skip the last 2 elements because the last one is reserved for VCC, and - // this is the 2nd to last element already. - for (MCPhysReg Reg : getAllSGPR128().drop_back(2).slice(NumPreloaded)) { - // Pick the first unallocated one. Make sure we don't clobber the other - // reserved input we needed. - if (!MRI.isPhysRegUsed(Reg)) { - assert(MRI.isAllocatable(Reg)); - MRI.replaceRegWith(ScratchRsrcReg, Reg); - ScratchRsrcReg = Reg; - MFI->setScratchRSrcReg(ScratchRsrcReg); - break; - } - } - } + DebugLoc DL; + MachineBasicBlock::iterator I = MBB.begin(); - if (ScratchWaveOffsetReg == TRI->reservedPrivateSegmentWaveByteOffsetReg(MF)) { - MachineRegisterInfo &MRI = MF.getRegInfo(); - unsigned NumPreloaded = MFI->getNumPreloadedSGPRs(); + // If we reserved the original input registers, we don't need to copy to the + // reserved registers. - // We need to drop register from the end of the list that we cannot use - // for the scratch wave offset. - // + 2 s102 and s103 do not exist on VI. - // + 2 for vcc - // + 2 for xnack_mask - // + 2 for flat_scratch - // + 4 for registers reserved for scratch resource register - // + 1 for register reserved for scratch wave offset. (By exluding this - // register from the list to consider, it means that when this - // register is being used for the scratch wave offset and there - // are no other free SGPRs, then the value will stay in this register. - // ---- - // 13 - for (MCPhysReg Reg : getAllSGPRs().drop_back(13).slice(NumPreloaded)) { - // Pick the first unallocated SGPR. Be careful not to pick an alias of the - // scratch descriptor, since we haven’t added its uses yet. - if (!MRI.isPhysRegUsed(Reg)) { - if (!MRI.isAllocatable(Reg) || - TRI->isSubRegisterEq(ScratchRsrcReg, Reg)) - continue; + bool CopyBuffer = ResourceRegUsed && + PreloadedPrivateBufferReg != AMDGPU::NoRegister && + ScratchRsrcReg != PreloadedPrivateBufferReg; - MRI.replaceRegWith(ScratchWaveOffsetReg, Reg); - ScratchWaveOffsetReg = Reg; - MFI->setScratchWaveOffsetReg(ScratchWaveOffsetReg); - break; - } - } - } + // This needs to be careful of the copying order to avoid overwriting one of + // the input registers before it's been copied to it's final + // destination. Usually the offset should be copied first. + bool CopyBufferFirst = TRI->isSubRegisterEq(PreloadedPrivateBufferReg, + ScratchWaveOffsetReg); + if (CopyBuffer && CopyBufferFirst) { + BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg) + .addReg(PreloadedPrivateBufferReg, RegState::Kill); } - - assert(!TRI->isSubRegister(ScratchRsrcReg, ScratchWaveOffsetReg)); - - const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32); - DebugLoc DL; - - if (PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg) { - // Make sure we emit the copy for the offset first. We may have chosen to copy - // the buffer resource into a register that aliases the input offset register. - BuildMI(MBB, I, DL, SMovB32, ScratchWaveOffsetReg) + if (OffsetRegUsed && + PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg) { + BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg) .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill); } - if (ST.isAmdHsaOS()) { - // Insert copies from argument register. - assert( - !TRI->isSubRegisterEq(PreloadedPrivateBufferReg, ScratchRsrcReg) && - !TRI->isSubRegisterEq(PreloadedPrivateBufferReg, ScratchWaveOffsetReg)); - - unsigned Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1); - unsigned Rsrc23 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2_sub3); - - unsigned Lo = TRI->getSubReg(PreloadedPrivateBufferReg, AMDGPU::sub0_sub1); - unsigned Hi = TRI->getSubReg(PreloadedPrivateBufferReg, AMDGPU::sub2_sub3); + if (CopyBuffer && !CopyBufferFirst) { + BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg) + .addReg(PreloadedPrivateBufferReg, RegState::Kill); + } - const MCInstrDesc &SMovB64 = TII->get(AMDGPU::S_MOV_B64); + if (ResourceRegUsed && PreloadedPrivateBufferReg == AMDGPU::NoRegister) { + assert(!ST.isAmdCodeObjectV2()); + const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32); - BuildMI(MBB, I, DL, SMovB64, Rsrc01) - .addReg(Lo, RegState::Kill); - BuildMI(MBB, I, DL, SMovB64, Rsrc23) - .addReg(Hi, RegState::Kill); - } else { unsigned Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0); unsigned Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1); unsigned Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2); @@ -260,15 +330,6 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, .addImm(Rsrc23 >> 32) .addReg(ScratchRsrcReg, RegState::ImplicitDefine); } - - // Make the register selected live throughout the function. - for (MachineBasicBlock &OtherBB : MF) { - if (&OtherBB == &MBB) - continue; - - OtherBB.addLiveIn(ScratchRsrcReg); - OtherBB.addLiveIn(ScratchWaveOffsetReg); - } } void SIFrameLowering::emitEpilogue(MachineFunction &MF, @@ -279,20 +340,20 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF, void SIFrameLowering::processFunctionBeforeFrameFinalized( MachineFunction &MF, RegScavenger *RS) const { - MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); - if (!MFI->hasStackObjects()) + if (!MFI.hasStackObjects()) return; - bool MayNeedScavengingEmergencySlot = MFI->hasStackObjects(); + bool MayNeedScavengingEmergencySlot = MFI.hasStackObjects(); assert((RS || !MayNeedScavengingEmergencySlot) && "RegScavenger required if spilling"); if (MayNeedScavengingEmergencySlot) { - int ScavengeFI = MFI->CreateSpillStackObject( + int ScavengeFI = MFI.CreateStackObject( AMDGPU::SGPR_32RegClass.getSize(), - AMDGPU::SGPR_32RegClass.getAlignment()); + AMDGPU::SGPR_32RegClass.getAlignment(), false); RS->addScavengingFrameIndex(ScavengeFI); } } |
