diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 498 |
1 files changed, 461 insertions, 37 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp index 3caa75e4d958..71be73c2f0e4 100644 --- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp +++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -84,22 +84,6 @@ enum class SIAtomicAddrSpace { LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL) }; -/// Sets named bit \p BitName to "true" if present in instruction \p MI. -/// \returns Returns true if \p MI is modified, false otherwise. -template <uint16_t BitName> -bool enableNamedBit(const MachineBasicBlock::iterator &MI) { - int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName); - if (BitIdx == -1) - return false; - - MachineOperand &Bit = MI->getOperand(BitIdx); - if (Bit.getImm() != 0) - return false; - - Bit.setImm(1); - return true; -} - class SIMemOpInfo final { private: @@ -129,12 +113,43 @@ private: IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering), IsVolatile(IsVolatile), IsNonTemporal(IsNonTemporal) { + + if (Ordering == AtomicOrdering::NotAtomic) { + assert(Scope == SIAtomicScope::NONE && + OrderingAddrSpace == SIAtomicAddrSpace::NONE && + !IsCrossAddressSpaceOrdering && + FailureOrdering == AtomicOrdering::NotAtomic); + return; + } + + assert(Scope != SIAtomicScope::NONE && + (OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != + SIAtomicAddrSpace::NONE && + (InstrAddrSpace & SIAtomicAddrSpace::ATOMIC) != + SIAtomicAddrSpace::NONE && + !isStrongerThan(FailureOrdering, Ordering)); + // There is also no cross address space ordering if the ordering // address space is the same as the instruction address space and // only contains a single address space. if ((OrderingAddrSpace == InstrAddrSpace) && isPowerOf2_32(uint32_t(InstrAddrSpace))) this->IsCrossAddressSpaceOrdering = false; + + // Limit the scope to the maximum supported by the instruction's address + // spaces. + if ((InstrAddrSpace & ~SIAtomicAddrSpace::SCRATCH) == + SIAtomicAddrSpace::NONE) { + this->Scope = std::min(Scope, SIAtomicScope::SINGLETHREAD); + } else if ((InstrAddrSpace & + ~(SIAtomicAddrSpace::SCRATCH | SIAtomicAddrSpace::LDS)) == + SIAtomicAddrSpace::NONE) { + this->Scope = std::min(Scope, SIAtomicScope::WORKGROUP); + } else if ((InstrAddrSpace & + ~(SIAtomicAddrSpace::SCRATCH | SIAtomicAddrSpace::LDS | + SIAtomicAddrSpace::GDS)) == SIAtomicAddrSpace::NONE) { + this->Scope = std::min(Scope, SIAtomicScope::AGENT); + } } public: @@ -202,12 +217,12 @@ private: void reportUnsupported(const MachineBasicBlock::iterator &MI, const char *Msg) const; - /// Inspects the target synchonization scope \p SSID and determines + /// Inspects the target synchronization scope \p SSID and determines /// the SI atomic scope it corresponds to, the address spaces it /// covers, and whether the memory ordering applies between address /// spaces. Optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>> - toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrScope) const; + toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrAddrSpace) const; /// \return Return a bit set of the address spaces accessed by \p AS. SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) const; @@ -257,6 +272,11 @@ protected: SICacheControl(const GCNSubtarget &ST); + /// Sets named bit \p BitName to "true" if present in instruction \p MI. + /// \returns Returns true if \p MI is modified, false otherwise. + bool enableNamedBit(const MachineBasicBlock::iterator MI, + AMDGPU::CPol::CPol Bit) const; + public: /// Create a cache control for the subtarget \p ST. @@ -269,6 +289,20 @@ public: SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace) const = 0; + /// Update \p MI memory store instruction to bypass any caches up to + /// the \p Scope memory scope for address spaces \p + /// AddrSpace. Return true iff the instruction was modified. + virtual bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const = 0; + + /// Update \p MI memory read-modify-write instruction to bypass any caches up + /// to the \p Scope memory scope for address spaces \p AddrSpace. Return true + /// iff the instruction was modified. + virtual bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const = 0; + /// Update \p MI memory instruction of kind \p Op associated with address /// spaces \p AddrSpace to indicate it is volatile and/or nontemporal. Return /// true iff the instruction was modified. @@ -324,13 +358,13 @@ protected: /// Sets GLC bit to "true" if present in \p MI. Returns true if \p MI /// is modified, false otherwise. bool enableGLCBit(const MachineBasicBlock::iterator &MI) const { - return enableNamedBit<AMDGPU::OpName::glc>(MI); + return enableNamedBit(MI, AMDGPU::CPol::GLC); } /// Sets SLC bit to "true" if present in \p MI. Returns true if \p MI /// is modified, false otherwise. bool enableSLCBit(const MachineBasicBlock::iterator &MI) const { - return enableNamedBit<AMDGPU::OpName::slc>(MI); + return enableNamedBit(MI, AMDGPU::CPol::SLC); } public: @@ -341,6 +375,14 @@ public: SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace) const override; + bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + + bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, bool IsVolatile, @@ -377,13 +419,54 @@ public: }; +class SIGfx90ACacheControl : public SIGfx7CacheControl { +public: + + SIGfx90ACacheControl(const GCNSubtarget &ST) : SIGfx7CacheControl(ST) {}; + + bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + + bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + + bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + + bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, + SIAtomicAddrSpace AddrSpace, SIMemOp Op, + bool IsVolatile, + bool IsNonTemporal) const override; + + bool insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const override; + + bool insertAcquire(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const override; + + bool insertRelease(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + bool IsCrossAddrSpaceOrdering, + Position Pos) const override; +}; + class SIGfx10CacheControl : public SIGfx7CacheControl { protected: /// Sets DLC bit to "true" if present in \p MI. Returns true if \p MI /// is modified, false otherwise. bool enableDLCBit(const MachineBasicBlock::iterator &MI) const { - return enableNamedBit<AMDGPU::OpName::dlc>(MI); + return enableNamedBit(MI, AMDGPU::CPol::DLC); } public: @@ -424,7 +507,7 @@ private: /// Return true iff instruction \p MI is a atomic instruction that /// returns a result. bool isAtomicRet(const MachineInstr &MI) const { - return AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1; + return SIInstrInfo::isAtomicRet(MI); } /// Removes all processed atomic pseudo instructions from the current @@ -476,7 +559,7 @@ void SIMemOpAccess::reportUnsupported(const MachineBasicBlock::iterator &MI, Optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>> SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, - SIAtomicAddrSpace InstrScope) const { + SIAtomicAddrSpace InstrAddrSpace) const { if (SSID == SyncScope::System) return std::make_tuple(SIAtomicScope::SYSTEM, SIAtomicAddrSpace::ATOMIC, @@ -499,23 +582,23 @@ SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, true); if (SSID == MMI->getSystemOneAddressSpaceSSID()) return std::make_tuple(SIAtomicScope::SYSTEM, - SIAtomicAddrSpace::ATOMIC & InstrScope, + SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); if (SSID == MMI->getAgentOneAddressSpaceSSID()) return std::make_tuple(SIAtomicScope::AGENT, - SIAtomicAddrSpace::ATOMIC & InstrScope, + SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); if (SSID == MMI->getWorkgroupOneAddressSpaceSSID()) return std::make_tuple(SIAtomicScope::WORKGROUP, - SIAtomicAddrSpace::ATOMIC & InstrScope, + SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); if (SSID == MMI->getWavefrontOneAddressSpaceSSID()) return std::make_tuple(SIAtomicScope::WAVEFRONT, - SIAtomicAddrSpace::ATOMIC & InstrScope, + SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); if (SSID == MMI->getSingleThreadOneAddressSpaceSSID()) return std::make_tuple(SIAtomicScope::SINGLETHREAD, - SIAtomicAddrSpace::ATOMIC & InstrScope, + SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); return None; } @@ -557,7 +640,7 @@ Optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO( IsVolatile |= MMO->isVolatile(); InstrAddrSpace |= toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace()); - AtomicOrdering OpOrdering = MMO->getOrdering(); + AtomicOrdering OpOrdering = MMO->getSuccessOrdering(); if (OpOrdering != AtomicOrdering::NotAtomic) { const auto &IsSyncScopeInclusion = MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID()); @@ -568,9 +651,9 @@ Optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO( } SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID(); - Ordering = - isStrongerThan(Ordering, OpOrdering) ? - Ordering : MMO->getOrdering(); + Ordering = isStrongerThan(Ordering, OpOrdering) + ? Ordering + : MMO->getSuccessOrdering(); assert(MMO->getFailureOrdering() != AtomicOrdering::Release && MMO->getFailureOrdering() != AtomicOrdering::AcquireRelease); FailureOrdering = @@ -591,7 +674,8 @@ Optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO( std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) = ScopeOrNone.getValue(); if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) || - ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace)) { + ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace) || + ((InstrAddrSpace & SIAtomicAddrSpace::ATOMIC) == SIAtomicAddrSpace::NONE)) { reportUnsupported(MI, "Unsupported atomic address space"); return None; } @@ -659,7 +743,7 @@ Optional<SIMemOpInfo> SIMemOpAccess::getAtomicFenceInfo( } return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, SIAtomicAddrSpace::ATOMIC, - IsCrossAddressSpaceOrdering); + IsCrossAddressSpaceOrdering, AtomicOrdering::NotAtomic); } Optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo( @@ -682,9 +766,21 @@ SICacheControl::SICacheControl(const GCNSubtarget &ST) : ST(ST) { InsertCacheInv = !AmdgcnSkipCacheInvalidations; } +bool SICacheControl::enableNamedBit(const MachineBasicBlock::iterator MI, + AMDGPU::CPol::CPol Bit) const { + MachineOperand *CPol = TII->getNamedOperand(*MI, AMDGPU::OpName::cpol); + if (!CPol) + return false; + + CPol->setImm(CPol->getImm() | Bit); + return true; +} + /* static */ std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST) { GCNSubtarget::Generation Generation = ST.getGeneration(); + if (ST.hasGFX90AInsts()) + return std::make_unique<SIGfx90ACacheControl>(ST); if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS) return std::make_unique<SIGfx6CacheControl>(ST); if (Generation < AMDGPUSubtarget::GFX10) @@ -725,6 +821,32 @@ bool SIGfx6CacheControl::enableLoadCacheBypass( return Changed; } +bool SIGfx6CacheControl::enableStoreCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(!MI->mayLoad() && MI->mayStore()); + bool Changed = false; + + /// The L1 cache is write through so does not need to be bypassed. There is no + /// bypass control for the L2 cache at the isa level. + + return Changed; +} + +bool SIGfx6CacheControl::enableRMWCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(MI->mayLoad() && MI->mayStore()); + bool Changed = false; + + /// The L1 cache is write through so does not need to be bypassed. There is no + /// bypass control for the L2 cache at the isa level. + + return Changed; +} + bool SIGfx6CacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, bool IsVolatile, bool IsNonTemporal) const { @@ -968,6 +1090,292 @@ bool SIGfx7CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, return Changed; } +bool SIGfx90ACacheControl::enableLoadCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(MI->mayLoad() && !MI->mayStore()); + bool Changed = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + Changed |= enableGLCBit(MI); + break; + case SIAtomicScope::WORKGROUP: + // In threadgroup split mode the waves of a work-group can be executing on + // different CUs. Therefore need to bypass the L1 which is per CU. + // Otherwise in non-threadgroup split mode all waves of a work-group are + // on the same CU, and so the L1 does not need to be bypassed. + if (ST.isTgSplitEnabled()) Changed |= enableGLCBit(MI); + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to bypass. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory caches + /// to be bypassed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not have a cache. + + return Changed; +} + +bool SIGfx90ACacheControl::enableStoreCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(!MI->mayLoad() && MI->mayStore()); + bool Changed = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + /// Do not set glc for store atomic operations as they implicitly write + /// through the L1 cache. + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to bypass. Store atomics implicitly write through the L1 + // cache. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory caches + /// to be bypassed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not have a cache. + + return Changed; +} + +bool SIGfx90ACacheControl::enableRMWCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(MI->mayLoad() && MI->mayStore()); + bool Changed = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + /// Do not set glc for RMW atomic operations as they implicitly bypass + /// the L1 cache, and the glc bit is instead used to indicate if they are + /// return or no-return. + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to bypass. RMW atomics implicitly bypass the L1 cache. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + return Changed; +} + +bool SIGfx90ACacheControl::enableVolatileAndOrNonTemporal( + MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, + bool IsVolatile, bool IsNonTemporal) const { + // Only handle load and store, not atomic read-modify-write insructions. The + // latter use glc to indicate if the atomic returns a result and so must not + // be used for cache control. + assert(MI->mayLoad() ^ MI->mayStore()); + + // Only update load and store, not LLVM IR atomic read-modify-write + // instructions. The latter are always marked as volatile so cannot sensibly + // handle it as do not want to pessimize all atomics. Also they do not support + // the nontemporal attribute. + assert( Op == SIMemOp::LOAD || Op == SIMemOp::STORE); + + bool Changed = false; + + if (IsVolatile) { + if (Op == SIMemOp::LOAD) { + Changed |= enableGLCBit(MI); + } + + // Ensure operation has completed at system scope to cause all volatile + // operations to be visible outside the program in a global order. Do not + // request cross address space as only the global address space can be + // observable outside the program, so no need to cause a waitcnt for LDS + // address space operations. + Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, + Position::AFTER); + + return Changed; + } + + if (IsNonTemporal) { + // Request L1 MISS_EVICT and L2 STREAM for load and store instructions. + Changed |= enableGLCBit(MI); + Changed |= enableSLCBit(MI); + return Changed; + } + + return Changed; +} + +bool SIGfx90ACacheControl::insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const { + if (ST.isTgSplitEnabled()) { + // In threadgroup split mode the waves of a work-group can be executing on + // different CUs. Therefore need to wait for global or GDS memory operations + // to complete to ensure they are visible to waves in the other CUs. + // Otherwise in non-threadgroup split mode all waves of a work-group are on + // the same CU, so no need to wait for global memory as all waves in the + // work-group access the same the L1, nor wait for GDS as access are ordered + // on a CU. + if (((AddrSpace & (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH | + SIAtomicAddrSpace::GDS)) != SIAtomicAddrSpace::NONE) && + (Scope == SIAtomicScope::WORKGROUP)) { + // Same as GFX7 using agent scope. + Scope = SIAtomicScope::AGENT; + } + // In threadgroup split mode LDS cannot be allocated so no need to wait for + // LDS memory operations. + AddrSpace &= ~SIAtomicAddrSpace::LDS; + } + return SIGfx7CacheControl::insertWait(MI, Scope, AddrSpace, Op, + IsCrossAddrSpaceOrdering, Pos); +} + +bool SIGfx90ACacheControl::insertAcquire(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const { + if (!InsertCacheInv) + return false; + + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (Pos == Position::AFTER) + ++MI; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + // Ensures that following loads will not see stale remote VMEM data or + // stale local VMEM data with MTYPE NC. Local VMEM data with MTYPE RW and + // CC will never be stale due to the local memory probes. + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_INVL2)); + // Inserting a "S_WAITCNT vmcnt(0)" after is not required because the + // hardware does not reorder memory operations by the same wave with + // respect to a preceding "BUFFER_INVL2". The invalidate is guaranteed to + // remove any cache lines of earlier writes by the same wave and ensures + // later reads by the same wave will refetch the cache lines. + Changed = true; + break; + case SIAtomicScope::AGENT: + // Same as GFX7. + break; + case SIAtomicScope::WORKGROUP: + // In threadgroup split mode the waves of a work-group can be executing on + // different CUs. Therefore need to invalidate the L1 which is per CU. + // Otherwise in non-threadgroup split mode all waves of a work-group are + // on the same CU, and so the L1 does not need to be invalidated. + if (ST.isTgSplitEnabled()) { + // Same as GFX7 using agent scope. + Scope = SIAtomicScope::AGENT; + } + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // Same as GFX7. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory cache + /// to be flushed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not have a cache. + + if (Pos == Position::AFTER) + --MI; + + Changed |= SIGfx7CacheControl::insertAcquire(MI, Scope, AddrSpace, Pos); + + return Changed; +} + +bool SIGfx90ACacheControl::insertRelease(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + bool IsCrossAddrSpaceOrdering, + Position Pos) const { + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (Pos == Position::AFTER) + ++MI; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + // Inserting a "S_WAITCNT vmcnt(0)" before is not required because the + // hardware does not reorder memory operations by the same wave with + // respect to a following "BUFFER_WBL2". The "BUFFER_WBL2" is guaranteed + // to initiate writeback of any dirty cache lines of earlier writes by the + // same wave. A "S_WAITCNT vmcnt(0)" is needed after to ensure the + // writeback has completed. + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBL2)); + // Followed by same as GFX7, which will ensure the necessary "S_WAITCNT + // vmcnt(0)" needed by the "BUFFER_WBL2". + Changed = true; + break; + case SIAtomicScope::AGENT: + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // Same as GFX7. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if (Pos == Position::AFTER) + --MI; + + Changed |= + SIGfx7CacheControl::insertRelease(MI, Scope, AddrSpace, + IsCrossAddrSpaceOrdering, Pos); + + return Changed; +} + bool SIGfx10CacheControl::enableLoadCacheBypass( const MachineBasicBlock::iterator &MI, SIAtomicScope Scope, @@ -1292,6 +1700,13 @@ bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI, bool Changed = false; if (MOI.isAtomic()) { + if (MOI.getOrdering() == AtomicOrdering::Monotonic || + MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { + Changed |= CC->enableStoreCacheBypass(MI, MOI.getScope(), + MOI.getOrderingAddrSpace()); + } + if (MOI.getOrdering() == AtomicOrdering::Release || MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= CC->insertRelease(MI, MOI.getScope(), @@ -1336,7 +1751,7 @@ bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI, Position::BEFORE); // TODO: If both release and invalidate are happening they could be combined - // to use the single "BUFFER_WBL2" instruction. This could be done by + // to use the single "BUFFER_WBINV*" instruction. This could be done by // reorganizing this code or as part of optimizing SIInsertWaitcnt pass to // track cache invalidate and write back instructions. @@ -1360,6 +1775,15 @@ bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, bool Changed = false; if (MOI.isAtomic()) { + if (MOI.getOrdering() == AtomicOrdering::Monotonic || + MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { + Changed |= CC->enableRMWCacheBypass(MI, MOI.getScope(), + MOI.getInstrAddrSpace()); + } + if (MOI.getOrdering() == AtomicOrdering::Release || MOI.getOrdering() == AtomicOrdering::AcquireRelease || MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || @@ -1375,7 +1799,7 @@ bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, MOI.getFailureOrdering() == AtomicOrdering::Acquire || MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) { Changed |= CC->insertWait(MI, MOI.getScope(), - MOI.getOrderingAddrSpace(), + MOI.getInstrAddrSpace(), isAtomicRet(*MI) ? SIMemOp::LOAD : SIMemOp::STORE, MOI.getIsCrossAddressSpaceOrdering(), @@ -1401,7 +1825,7 @@ bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) { for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) { // Unbundle instructions after the post-RA scheduler. - if (MI->isBundle()) { + if (MI->isBundle() && MI->mayLoadOrStore()) { MachineBasicBlock::instr_iterator II(MI->getIterator()); for (MachineBasicBlock::instr_iterator I = ++II, E = MBB.instr_end(); I != E && I->isBundledWithPred(); ++I) { |