From 0b57cec536236d46e3dba9bd041533462f33dbb7 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Fri, 20 Dec 2019 19:53:05 +0000 Subject: Move all sources from the llvm project into contrib/llvm-project. This uses the new layout of the upstream repository, which was recently migrated to GitHub, and converted into a "monorepo". That is, most of the earlier separate sub-projects with their own branches and tags were consolidated into one top-level directory, and are now branched and tagged together. Updating the vendor area to match this layout is next. --- .../llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 1317 ++++++++++++++++++++ 1 file changed, 1317 insertions(+) create mode 100644 contrib/llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp (limited to 'contrib/llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp') diff --git a/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp new file mode 100644 index 000000000000..4320e6c957a0 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -0,0 +1,1317 @@ +//===- SIMemoryLegalizer.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// Memory legalizer - implements memory model. More information can be +/// found here: +/// http://llvm.org/docs/AMDGPUUsage.html#memory-model +// +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUMachineModuleInfo.h" +#include "AMDGPUSubtarget.h" +#include "SIDefines.h" +#include "SIInstrInfo.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "Utils/AMDGPUBaseInfo.h" +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/Pass.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/MathExtras.h" +#include +#include + +using namespace llvm; +using namespace llvm::AMDGPU; + +#define DEBUG_TYPE "si-memory-legalizer" +#define PASS_NAME "SI Memory Legalizer" + +namespace { + +LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); + +/// Memory operation flags. Can be ORed together. +enum class SIMemOp { + NONE = 0u, + LOAD = 1u << 0, + STORE = 1u << 1, + LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ STORE) +}; + +/// Position to insert a new instruction relative to an existing +/// instruction. +enum class Position { + BEFORE, + AFTER +}; + +/// The atomic synchronization scopes supported by the AMDGPU target. +enum class SIAtomicScope { + NONE, + SINGLETHREAD, + WAVEFRONT, + WORKGROUP, + AGENT, + SYSTEM +}; + +/// The distinct address spaces supported by the AMDGPU target for +/// atomic memory operation. Can be ORed toether. +enum class SIAtomicAddrSpace { + NONE = 0u, + GLOBAL = 1u << 0, + LDS = 1u << 1, + SCRATCH = 1u << 2, + GDS = 1u << 3, + OTHER = 1u << 4, + + /// The address spaces that can be accessed by a FLAT instruction. + FLAT = GLOBAL | LDS | SCRATCH, + + /// The address spaces that support atomic instructions. + ATOMIC = GLOBAL | LDS | SCRATCH | GDS, + + /// All address spaces. + ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER, + + LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL) +}; + +/// Sets named bit \p BitName to "true" if present in instruction \p MI. +/// \returns Returns true if \p MI is modified, false otherwise. +template +bool enableNamedBit(const MachineBasicBlock::iterator &MI) { + int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName); + if (BitIdx == -1) + return false; + + MachineOperand &Bit = MI->getOperand(BitIdx); + if (Bit.getImm() != 0) + return false; + + Bit.setImm(1); + return true; +} + +class SIMemOpInfo final { +private: + + friend class SIMemOpAccess; + + AtomicOrdering Ordering = AtomicOrdering::NotAtomic; + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic; + SIAtomicScope Scope = SIAtomicScope::SYSTEM; + SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE; + SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE; + bool IsCrossAddressSpaceOrdering = false; + bool IsNonTemporal = false; + + SIMemOpInfo(AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent, + SIAtomicScope Scope = SIAtomicScope::SYSTEM, + SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC, + SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL, + bool IsCrossAddressSpaceOrdering = true, + AtomicOrdering FailureOrdering = + AtomicOrdering::SequentiallyConsistent, + bool IsNonTemporal = false) + : Ordering(Ordering), FailureOrdering(FailureOrdering), + Scope(Scope), OrderingAddrSpace(OrderingAddrSpace), + InstrAddrSpace(InstrAddrSpace), + IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering), + IsNonTemporal(IsNonTemporal) { + // There is also no cross address space ordering if the ordering + // address space is the same as the instruction address space and + // only contains a single address space. + if ((OrderingAddrSpace == InstrAddrSpace) && + isPowerOf2_32(uint32_t(InstrAddrSpace))) + this->IsCrossAddressSpaceOrdering = false; + } + +public: + /// \returns Atomic synchronization scope of the machine instruction used to + /// create this SIMemOpInfo. + SIAtomicScope getScope() const { + return Scope; + } + + /// \returns Ordering constraint of the machine instruction used to + /// create this SIMemOpInfo. + AtomicOrdering getOrdering() const { + return Ordering; + } + + /// \returns Failure ordering constraint of the machine instruction used to + /// create this SIMemOpInfo. + AtomicOrdering getFailureOrdering() const { + return FailureOrdering; + } + + /// \returns The address spaces be accessed by the machine + /// instruction used to create this SiMemOpInfo. + SIAtomicAddrSpace getInstrAddrSpace() const { + return InstrAddrSpace; + } + + /// \returns The address spaces that must be ordered by the machine + /// instruction used to create this SiMemOpInfo. + SIAtomicAddrSpace getOrderingAddrSpace() const { + return OrderingAddrSpace; + } + + /// \returns Return true iff memory ordering of operations on + /// different address spaces is required. + bool getIsCrossAddressSpaceOrdering() const { + return IsCrossAddressSpaceOrdering; + } + + /// \returns True if memory access of the machine instruction used to + /// create this SIMemOpInfo is non-temporal, false otherwise. + bool isNonTemporal() const { + return IsNonTemporal; + } + + /// \returns True if ordering constraint of the machine instruction used to + /// create this SIMemOpInfo is unordered or higher, false otherwise. + bool isAtomic() const { + return Ordering != AtomicOrdering::NotAtomic; + } + +}; + +class SIMemOpAccess final { +private: + AMDGPUMachineModuleInfo *MMI = nullptr; + + /// Reports unsupported message \p Msg for \p MI to LLVM context. + void reportUnsupported(const MachineBasicBlock::iterator &MI, + const char *Msg) const; + + /// Inspects the target synchonization scope \p SSID and determines + /// the SI atomic scope it corresponds to, the address spaces it + /// covers, and whether the memory ordering applies between address + /// spaces. + Optional> + toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrScope) const; + + /// \return Return a bit set of the address spaces accessed by \p AS. + SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) const; + + /// \returns Info constructed from \p MI, which has at least machine memory + /// operand. + Optional constructFromMIWithMMO( + const MachineBasicBlock::iterator &MI) const; + +public: + /// Construct class to support accessing the machine memory operands + /// of instructions in the machine function \p MF. + SIMemOpAccess(MachineFunction &MF); + + /// \returns Load info if \p MI is a load operation, "None" otherwise. + Optional getLoadInfo( + const MachineBasicBlock::iterator &MI) const; + + /// \returns Store info if \p MI is a store operation, "None" otherwise. + Optional getStoreInfo( + const MachineBasicBlock::iterator &MI) const; + + /// \returns Atomic fence info if \p MI is an atomic fence operation, + /// "None" otherwise. + Optional getAtomicFenceInfo( + const MachineBasicBlock::iterator &MI) const; + + /// \returns Atomic cmpxchg/rmw info if \p MI is an atomic cmpxchg or + /// rmw operation, "None" otherwise. + Optional getAtomicCmpxchgOrRmwInfo( + const MachineBasicBlock::iterator &MI) const; +}; + +class SICacheControl { +protected: + + /// Instruction info. + const SIInstrInfo *TII = nullptr; + + IsaVersion IV; + + SICacheControl(const GCNSubtarget &ST); + +public: + + /// Create a cache control for the subtarget \p ST. + static std::unique_ptr create(const GCNSubtarget &ST); + + /// Update \p MI memory load instruction to bypass any caches up to + /// the \p Scope memory scope for address spaces \p + /// AddrSpace. Return true iff the instruction was modified. + virtual bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const = 0; + + /// Update \p MI memory instruction to indicate it is + /// nontemporal. Return true iff the instruction was modified. + virtual bool enableNonTemporal(const MachineBasicBlock::iterator &MI) + const = 0; + + /// Inserts any necessary instructions at position \p Pos relative + /// to instruction \p MI to ensure any caches associated with + /// address spaces \p AddrSpace for memory scopes up to memory scope + /// \p Scope are invalidated. Returns true iff any instructions + /// inserted. + virtual bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const = 0; + + /// Inserts any necessary instructions at position \p Pos relative + /// to instruction \p MI to ensure memory instructions of kind \p Op + /// associated with address spaces \p AddrSpace have completed as + /// observed by other memory instructions executing in memory scope + /// \p Scope. \p IsCrossAddrSpaceOrdering indicates if the memory + /// ordering is between address spaces. Returns true iff any + /// instructions inserted. + virtual bool insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const = 0; + + /// Virtual destructor to allow derivations to be deleted. + virtual ~SICacheControl() = default; + +}; + +class SIGfx6CacheControl : public SICacheControl { +protected: + + /// Sets GLC bit to "true" if present in \p MI. Returns true if \p MI + /// is modified, false otherwise. + bool enableGLCBit(const MachineBasicBlock::iterator &MI) const { + return enableNamedBit(MI); + } + + /// Sets SLC bit to "true" if present in \p MI. Returns true if \p MI + /// is modified, false otherwise. + bool enableSLCBit(const MachineBasicBlock::iterator &MI) const { + return enableNamedBit(MI); + } + +public: + + SIGfx6CacheControl(const GCNSubtarget &ST) : SICacheControl(ST) {}; + + bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + + bool enableNonTemporal(const MachineBasicBlock::iterator &MI) const override; + + bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const override; + + bool insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const override; +}; + +class SIGfx7CacheControl : public SIGfx6CacheControl { +public: + + SIGfx7CacheControl(const GCNSubtarget &ST) : SIGfx6CacheControl(ST) {}; + + bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const override; + +}; + +class SIGfx10CacheControl : public SIGfx7CacheControl { +protected: + bool CuMode = false; + + /// Sets DLC bit to "true" if present in \p MI. Returns true if \p MI + /// is modified, false otherwise. + bool enableDLCBit(const MachineBasicBlock::iterator &MI) const { + return enableNamedBit(MI); + } + +public: + + SIGfx10CacheControl(const GCNSubtarget &ST, bool CuMode) : + SIGfx7CacheControl(ST), CuMode(CuMode) {}; + + bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const override; + + bool enableNonTemporal(const MachineBasicBlock::iterator &MI) const override; + + bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const override; + + bool insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const override; +}; + +class SIMemoryLegalizer final : public MachineFunctionPass { +private: + + /// Cache Control. + std::unique_ptr CC = nullptr; + + /// List of atomic pseudo instructions. + std::list AtomicPseudoMIs; + + /// Return true iff instruction \p MI is a atomic instruction that + /// returns a result. + bool isAtomicRet(const MachineInstr &MI) const { + return AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1; + } + + /// Removes all processed atomic pseudo instructions from the current + /// function. Returns true if current function is modified, false otherwise. + bool removeAtomicPseudoMIs(); + + /// Expands load operation \p MI. Returns true if instructions are + /// added/deleted or \p MI is modified, false otherwise. + bool expandLoad(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI); + /// Expands store operation \p MI. Returns true if instructions are + /// added/deleted or \p MI is modified, false otherwise. + bool expandStore(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI); + /// Expands atomic fence operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicFence(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI); + /// Expands atomic cmpxchg or rmw operation \p MI. Returns true if + /// instructions are added/deleted or \p MI is modified, false otherwise. + bool expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI); + +public: + static char ID; + + SIMemoryLegalizer() : MachineFunctionPass(ID) {} + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { + return PASS_NAME; + } + + bool runOnMachineFunction(MachineFunction &MF) override; +}; + +} // end namespace anonymous + +void SIMemOpAccess::reportUnsupported(const MachineBasicBlock::iterator &MI, + const char *Msg) const { + const Function &Func = MI->getParent()->getParent()->getFunction(); + DiagnosticInfoUnsupported Diag(Func, Msg, MI->getDebugLoc()); + Func.getContext().diagnose(Diag); +} + +Optional> +SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, + SIAtomicAddrSpace InstrScope) const { + if (SSID == SyncScope::System) + return std::make_tuple(SIAtomicScope::SYSTEM, + SIAtomicAddrSpace::ATOMIC, + true); + if (SSID == MMI->getAgentSSID()) + return std::make_tuple(SIAtomicScope::AGENT, + SIAtomicAddrSpace::ATOMIC, + true); + if (SSID == MMI->getWorkgroupSSID()) + return std::make_tuple(SIAtomicScope::WORKGROUP, + SIAtomicAddrSpace::ATOMIC, + true); + if (SSID == MMI->getWavefrontSSID()) + return std::make_tuple(SIAtomicScope::WAVEFRONT, + SIAtomicAddrSpace::ATOMIC, + true); + if (SSID == SyncScope::SingleThread) + return std::make_tuple(SIAtomicScope::SINGLETHREAD, + SIAtomicAddrSpace::ATOMIC, + true); + if (SSID == MMI->getSystemOneAddressSpaceSSID()) + return std::make_tuple(SIAtomicScope::SYSTEM, + SIAtomicAddrSpace::ATOMIC & InstrScope, + false); + if (SSID == MMI->getAgentOneAddressSpaceSSID()) + return std::make_tuple(SIAtomicScope::AGENT, + SIAtomicAddrSpace::ATOMIC & InstrScope, + false); + if (SSID == MMI->getWorkgroupOneAddressSpaceSSID()) + return std::make_tuple(SIAtomicScope::WORKGROUP, + SIAtomicAddrSpace::ATOMIC & InstrScope, + false); + if (SSID == MMI->getWavefrontOneAddressSpaceSSID()) + return std::make_tuple(SIAtomicScope::WAVEFRONT, + SIAtomicAddrSpace::ATOMIC & InstrScope, + false); + if (SSID == MMI->getSingleThreadOneAddressSpaceSSID()) + return std::make_tuple(SIAtomicScope::SINGLETHREAD, + SIAtomicAddrSpace::ATOMIC & InstrScope, + false); + return None; +} + +SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const { + if (AS == AMDGPUAS::FLAT_ADDRESS) + return SIAtomicAddrSpace::FLAT; + if (AS == AMDGPUAS::GLOBAL_ADDRESS) + return SIAtomicAddrSpace::GLOBAL; + if (AS == AMDGPUAS::LOCAL_ADDRESS) + return SIAtomicAddrSpace::LDS; + if (AS == AMDGPUAS::PRIVATE_ADDRESS) + return SIAtomicAddrSpace::SCRATCH; + if (AS == AMDGPUAS::REGION_ADDRESS) + return SIAtomicAddrSpace::GDS; + + return SIAtomicAddrSpace::OTHER; +} + +SIMemOpAccess::SIMemOpAccess(MachineFunction &MF) { + MMI = &MF.getMMI().getObjFileInfo(); +} + +Optional SIMemOpAccess::constructFromMIWithMMO( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getNumMemOperands() > 0); + + SyncScope::ID SSID = SyncScope::SingleThread; + AtomicOrdering Ordering = AtomicOrdering::NotAtomic; + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic; + SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE; + bool IsNonTemporal = true; + + // Validator should check whether or not MMOs cover the entire set of + // locations accessed by the memory instruction. + for (const auto &MMO : MI->memoperands()) { + IsNonTemporal &= MMO->isNonTemporal(); + InstrAddrSpace |= + toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace()); + AtomicOrdering OpOrdering = MMO->getOrdering(); + if (OpOrdering != AtomicOrdering::NotAtomic) { + const auto &IsSyncScopeInclusion = + MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID()); + if (!IsSyncScopeInclusion) { + reportUnsupported(MI, + "Unsupported non-inclusive atomic synchronization scope"); + return None; + } + + SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID(); + Ordering = + isStrongerThan(Ordering, OpOrdering) ? + Ordering : MMO->getOrdering(); + assert(MMO->getFailureOrdering() != AtomicOrdering::Release && + MMO->getFailureOrdering() != AtomicOrdering::AcquireRelease); + FailureOrdering = + isStrongerThan(FailureOrdering, MMO->getFailureOrdering()) ? + FailureOrdering : MMO->getFailureOrdering(); + } + } + + SIAtomicScope Scope = SIAtomicScope::NONE; + SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE; + bool IsCrossAddressSpaceOrdering = false; + if (Ordering != AtomicOrdering::NotAtomic) { + auto ScopeOrNone = toSIAtomicScope(SSID, InstrAddrSpace); + if (!ScopeOrNone) { + reportUnsupported(MI, "Unsupported atomic synchronization scope"); + return None; + } + std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) = + ScopeOrNone.getValue(); + if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) || + ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace)) { + reportUnsupported(MI, "Unsupported atomic address space"); + return None; + } + } + return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace, + IsCrossAddressSpaceOrdering, FailureOrdering, IsNonTemporal); +} + +Optional SIMemOpAccess::getLoadInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(MI->mayLoad() && !MI->mayStore())) + return None; + + // Be conservative if there are no memory operands. + if (MI->getNumMemOperands() == 0) + return SIMemOpInfo(); + + return constructFromMIWithMMO(MI); +} + +Optional SIMemOpAccess::getStoreInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(!MI->mayLoad() && MI->mayStore())) + return None; + + // Be conservative if there are no memory operands. + if (MI->getNumMemOperands() == 0) + return SIMemOpInfo(); + + return constructFromMIWithMMO(MI); +} + +Optional SIMemOpAccess::getAtomicFenceInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE) + return None; + + AtomicOrdering Ordering = + static_cast(MI->getOperand(0).getImm()); + + SyncScope::ID SSID = static_cast(MI->getOperand(1).getImm()); + auto ScopeOrNone = toSIAtomicScope(SSID, SIAtomicAddrSpace::ATOMIC); + if (!ScopeOrNone) { + reportUnsupported(MI, "Unsupported atomic synchronization scope"); + return None; + } + + SIAtomicScope Scope = SIAtomicScope::NONE; + SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE; + bool IsCrossAddressSpaceOrdering = false; + std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) = + ScopeOrNone.getValue(); + + if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) || + ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace)) { + reportUnsupported(MI, "Unsupported atomic address space"); + return None; + } + + return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, SIAtomicAddrSpace::ATOMIC, + IsCrossAddressSpaceOrdering); +} + +Optional SIMemOpAccess::getAtomicCmpxchgOrRmwInfo( + const MachineBasicBlock::iterator &MI) const { + assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); + + if (!(MI->mayLoad() && MI->mayStore())) + return None; + + // Be conservative if there are no memory operands. + if (MI->getNumMemOperands() == 0) + return SIMemOpInfo(); + + return constructFromMIWithMMO(MI); +} + +SICacheControl::SICacheControl(const GCNSubtarget &ST) { + TII = ST.getInstrInfo(); + IV = getIsaVersion(ST.getCPU()); +} + +/* static */ +std::unique_ptr SICacheControl::create(const GCNSubtarget &ST) { + GCNSubtarget::Generation Generation = ST.getGeneration(); + if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS) + return make_unique(ST); + if (Generation < AMDGPUSubtarget::GFX10) + return make_unique(ST); + return make_unique(ST, ST.isCuModeEnabled()); +} + +bool SIGfx6CacheControl::enableLoadCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(MI->mayLoad() && !MI->mayStore()); + bool Changed = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + /// TODO: Do not set glc for rmw atomic operations as they + /// implicitly bypass the L1 cache. + + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + Changed |= enableGLCBit(MI); + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to bypass. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory caches + /// to be bypassed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not hava a cache. + + return Changed; +} + +bool SIGfx6CacheControl::enableNonTemporal( + const MachineBasicBlock::iterator &MI) const { + assert(MI->mayLoad() ^ MI->mayStore()); + bool Changed = false; + + /// TODO: Do not enableGLCBit if rmw atomic. + Changed |= enableGLCBit(MI); + Changed |= enableSLCBit(MI); + + return Changed; +} + +bool SIGfx6CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const { + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (Pos == Position::AFTER) + ++MI; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBINVL1)); + Changed = true; + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to invalidate. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory cache + /// to be flushed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not hava a cache. + + if (Pos == Position::AFTER) + --MI; + + return Changed; +} + +bool SIGfx6CacheControl::insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const { + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (Pos == Position::AFTER) + ++MI; + + bool VMCnt = false; + bool LGKMCnt = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + VMCnt |= true; + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // The L1 cache keeps all memory operations in order for + // wavefronts in the same work-group. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if ((AddrSpace & SIAtomicAddrSpace::LDS) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + case SIAtomicScope::WORKGROUP: + // If no cross address space ordering then an LDS waitcnt is not + // needed as LDS operations for all waves are executed in a + // total global ordering as observed by all waves. Required if + // also synchronizing with global/GDS memory as LDS operations + // could be reordered with respect to later global/GDS memory + // operations of the same wave. + LGKMCnt |= IsCrossAddrSpaceOrdering; + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // The LDS keeps all memory operations in order for + // the same wavesfront. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if ((AddrSpace & SIAtomicAddrSpace::GDS) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + // If no cross address space ordering then an GDS waitcnt is not + // needed as GDS operations for all waves are executed in a + // total global ordering as observed by all waves. Required if + // also synchronizing with global/LDS memory as GDS operations + // could be reordered with respect to later global/LDS memory + // operations of the same wave. + LGKMCnt |= IsCrossAddrSpaceOrdering; + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // The GDS keeps all memory operations in order for + // the same work-group. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if (VMCnt || LGKMCnt) { + unsigned WaitCntImmediate = + AMDGPU::encodeWaitcnt(IV, + VMCnt ? 0 : getVmcntBitMask(IV), + getExpcntBitMask(IV), + LGKMCnt ? 0 : getLgkmcntBitMask(IV)); + BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(WaitCntImmediate); + Changed = true; + } + + if (Pos == Position::AFTER) + --MI; + + return Changed; +} + +bool SIGfx7CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const { + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + const GCNSubtarget &STM = MBB.getParent()->getSubtarget(); + + const unsigned Flush = STM.isAmdPalOS() || STM.isMesa3DOS() + ? AMDGPU::BUFFER_WBINVL1 + : AMDGPU::BUFFER_WBINVL1_VOL; + + if (Pos == Position::AFTER) + ++MI; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + BuildMI(MBB, MI, DL, TII->get(Flush)); + Changed = true; + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to invalidate. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory cache + /// to be flushed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not hava a cache. + + if (Pos == Position::AFTER) + --MI; + + return Changed; +} + +bool SIGfx10CacheControl::enableLoadCacheBypass( + const MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace) const { + assert(MI->mayLoad() && !MI->mayStore()); + bool Changed = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + /// TODO Do not set glc for rmw atomic operations as they + /// implicitly bypass the L0/L1 caches. + + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + Changed |= enableGLCBit(MI); + Changed |= enableDLCBit(MI); + break; + case SIAtomicScope::WORKGROUP: + // In WGP mode the waves of a work-group can be executing on either CU of + // the WGP. Therefore need to bypass the L0 which is per CU. Otherwise in + // CU mode and all waves of a work-group are on the same CU, and so the + // L0 does not need to be bypassed. + if (!CuMode) Changed |= enableGLCBit(MI); + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to bypass. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory caches + /// to be bypassed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not hava a cache. + + return Changed; +} + +bool SIGfx10CacheControl::enableNonTemporal( + const MachineBasicBlock::iterator &MI) const { + assert(MI->mayLoad() ^ MI->mayStore()); + bool Changed = false; + + Changed |= enableSLCBit(MI); + /// TODO for store (non-rmw atomic) instructions also enableGLCBit(MI) + + return Changed; +} + +bool SIGfx10CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + Position Pos) const { + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (Pos == Position::AFTER) + ++MI; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV)); + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL1_INV)); + Changed = true; + break; + case SIAtomicScope::WORKGROUP: + // In WGP mode the waves of a work-group can be executing on either CU of + // the WGP. Therefore need to invalidate the L0 which is per CU. Otherwise + // in CU mode and all waves of a work-group are on the same CU, and so the + // L0 does not need to be invalidated. + if (!CuMode) { + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV)); + Changed = true; + } + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // No cache to invalidate. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + /// The scratch address space does not need the global memory cache + /// to be flushed as all memory operations by the same thread are + /// sequentially consistent, and no other thread can access scratch + /// memory. + + /// Other address spaces do not hava a cache. + + if (Pos == Position::AFTER) + --MI; + + return Changed; +} + +bool SIGfx10CacheControl::insertWait(MachineBasicBlock::iterator &MI, + SIAtomicScope Scope, + SIAtomicAddrSpace AddrSpace, + SIMemOp Op, + bool IsCrossAddrSpaceOrdering, + Position Pos) const { + bool Changed = false; + + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (Pos == Position::AFTER) + ++MI; + + bool VMCnt = false; + bool VSCnt = false; + bool LGKMCnt = false; + + if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + if ((Op & SIMemOp::LOAD) != SIMemOp::NONE) + VMCnt |= true; + if ((Op & SIMemOp::STORE) != SIMemOp::NONE) + VSCnt |= true; + break; + case SIAtomicScope::WORKGROUP: + // In WGP mode the waves of a work-group can be executing on either CU of + // the WGP. Therefore need to wait for operations to complete to ensure + // they are visible to waves in the other CU as the L0 is per CU. + // Otherwise in CU mode and all waves of a work-group are on the same CU + // which shares the same L0. + if (!CuMode) { + if ((Op & SIMemOp::LOAD) != SIMemOp::NONE) + VMCnt |= true; + if ((Op & SIMemOp::STORE) != SIMemOp::NONE) + VSCnt |= true; + } + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // The L0 cache keeps all memory operations in order for + // work-items in the same wavefront. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if ((AddrSpace & SIAtomicAddrSpace::LDS) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + case SIAtomicScope::WORKGROUP: + // If no cross address space ordering then an LDS waitcnt is not + // needed as LDS operations for all waves are executed in a + // total global ordering as observed by all waves. Required if + // also synchronizing with global/GDS memory as LDS operations + // could be reordered with respect to later global/GDS memory + // operations of the same wave. + LGKMCnt |= IsCrossAddrSpaceOrdering; + break; + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // The LDS keeps all memory operations in order for + // the same wavesfront. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if ((AddrSpace & SIAtomicAddrSpace::GDS) != SIAtomicAddrSpace::NONE) { + switch (Scope) { + case SIAtomicScope::SYSTEM: + case SIAtomicScope::AGENT: + // If no cross address space ordering then an GDS waitcnt is not + // needed as GDS operations for all waves are executed in a + // total global ordering as observed by all waves. Required if + // also synchronizing with global/LDS memory as GDS operations + // could be reordered with respect to later global/LDS memory + // operations of the same wave. + LGKMCnt |= IsCrossAddrSpaceOrdering; + break; + case SIAtomicScope::WORKGROUP: + case SIAtomicScope::WAVEFRONT: + case SIAtomicScope::SINGLETHREAD: + // The GDS keeps all memory operations in order for + // the same work-group. + break; + default: + llvm_unreachable("Unsupported synchronization scope"); + } + } + + if (VMCnt || LGKMCnt) { + unsigned WaitCntImmediate = + AMDGPU::encodeWaitcnt(IV, + VMCnt ? 0 : getVmcntBitMask(IV), + getExpcntBitMask(IV), + LGKMCnt ? 0 : getLgkmcntBitMask(IV)); + BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(WaitCntImmediate); + Changed = true; + } + + if (VSCnt) { + BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT)) + .addReg(AMDGPU::SGPR_NULL, RegState::Undef) + .addImm(0); + Changed = true; + } + + if (Pos == Position::AFTER) + --MI; + + return Changed; +} + +bool SIMemoryLegalizer::removeAtomicPseudoMIs() { + if (AtomicPseudoMIs.empty()) + return false; + + for (auto &MI : AtomicPseudoMIs) + MI->eraseFromParent(); + + AtomicPseudoMIs.clear(); + return true; +} + +bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI) { + assert(MI->mayLoad() && !MI->mayStore()); + + bool Changed = false; + + if (MOI.isAtomic()) { + if (MOI.getOrdering() == AtomicOrdering::Monotonic || + MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { + Changed |= CC->enableLoadCacheBypass(MI, MOI.getScope(), + MOI.getOrderingAddrSpace()); + } + + if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) + Changed |= CC->insertWait(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + SIMemOp::LOAD | SIMemOp::STORE, + MOI.getIsCrossAddressSpaceOrdering(), + Position::BEFORE); + + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { + Changed |= CC->insertWait(MI, MOI.getScope(), + MOI.getInstrAddrSpace(), + SIMemOp::LOAD, + MOI.getIsCrossAddressSpaceOrdering(), + Position::AFTER); + Changed |= CC->insertCacheInvalidate(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + Position::AFTER); + } + + return Changed; + } + + // Atomic instructions do not have the nontemporal attribute. + if (MOI.isNonTemporal()) { + Changed |= CC->enableNonTemporal(MI); + return Changed; + } + + return Changed; +} + +bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI) { + assert(!MI->mayLoad() && MI->mayStore()); + + bool Changed = false; + + if (MOI.isAtomic()) { + if (MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) + Changed |= CC->insertWait(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + SIMemOp::LOAD | SIMemOp::STORE, + MOI.getIsCrossAddressSpaceOrdering(), + Position::BEFORE); + + return Changed; + } + + // Atomic instructions do not have the nontemporal attribute. + if (MOI.isNonTemporal()) { + Changed |= CC->enableNonTemporal(MI); + return Changed; + } + + return Changed; +} + +bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI) { + assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE); + + AtomicPseudoMIs.push_back(MI); + bool Changed = false; + + if (MOI.isAtomic()) { + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) + /// TODO: This relies on a barrier always generating a waitcnt + /// for LDS to ensure it is not reordered with the completion of + /// the proceeding LDS operations. If barrier had a memory + /// ordering and memory scope, then library does not need to + /// generate a fence. Could add support in this file for + /// barrier. SIInsertWaitcnt.cpp could then stop unconditionally + /// adding waitcnt before a S_BARRIER. + Changed |= CC->insertWait(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + SIMemOp::LOAD | SIMemOp::STORE, + MOI.getIsCrossAddressSpaceOrdering(), + Position::BEFORE); + + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) + Changed |= CC->insertCacheInvalidate(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + Position::BEFORE); + + return Changed; + } + + return Changed; +} + +bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, + MachineBasicBlock::iterator &MI) { + assert(MI->mayLoad() && MI->mayStore()); + + bool Changed = false; + + if (MOI.isAtomic()) { + if (MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || + MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) + Changed |= CC->insertWait(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + SIMemOp::LOAD | SIMemOp::STORE, + MOI.getIsCrossAddressSpaceOrdering(), + Position::BEFORE); + + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || + MOI.getFailureOrdering() == AtomicOrdering::Acquire || + MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) { + Changed |= CC->insertWait(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + isAtomicRet(*MI) ? SIMemOp::LOAD : + SIMemOp::STORE, + MOI.getIsCrossAddressSpaceOrdering(), + Position::AFTER); + Changed |= CC->insertCacheInvalidate(MI, MOI.getScope(), + MOI.getOrderingAddrSpace(), + Position::AFTER); + } + + return Changed; + } + + return Changed; +} + +bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + + SIMemOpAccess MOA(MF); + CC = SICacheControl::create(MF.getSubtarget()); + + for (auto &MBB : MF) { + for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) { + if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic)) + continue; + + if (const auto &MOI = MOA.getLoadInfo(MI)) + Changed |= expandLoad(MOI.getValue(), MI); + else if (const auto &MOI = MOA.getStoreInfo(MI)) + Changed |= expandStore(MOI.getValue(), MI); + else if (const auto &MOI = MOA.getAtomicFenceInfo(MI)) + Changed |= expandAtomicFence(MOI.getValue(), MI); + else if (const auto &MOI = MOA.getAtomicCmpxchgOrRmwInfo(MI)) + Changed |= expandAtomicCmpxchgOrRmw(MOI.getValue(), MI); + } + } + + Changed |= removeAtomicPseudoMIs(); + return Changed; +} + +INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false) + +char SIMemoryLegalizer::ID = 0; +char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID; + +FunctionPass *llvm::createSIMemoryLegalizerPass() { + return new SIMemoryLegalizer(); +} -- cgit v1.2.3