diff options
Diffstat (limited to 'llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp')
| -rw-r--r-- | llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp | 363 |
1 files changed, 363 insertions, 0 deletions
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp new file mode 100644 index 000000000000..5cda6a07352d --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp @@ -0,0 +1,363 @@ +//===-- SPIRVLegalizePointerCast.cpp ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The LLVM IR has multiple legal patterns we cannot lower to Logical SPIR-V. +// This pass modifies such loads to have an IR we can directly lower to valid +// logical SPIR-V. +// OpenCL can avoid this because they rely on ptrcast, which is not supported +// by logical SPIR-V. +// +// This pass relies on the assign_ptr_type intrinsic to deduce the type of the +// pointed values, must replace all occurences of `ptrcast`. This is why +// unhandled cases are reported as unreachable: we MUST cover all cases. +// +// 1. Loading the first element of an array +// +// %array = [10 x i32] +// %value = load i32, ptr %array +// +// LLVM can skip the GEP instruction, and only request loading the first 4 +// bytes. In logical SPIR-V, we need an OpAccessChain to access the first +// element. This pass will add a getelementptr instruction before the load. +// +// +// 2. Implicit downcast from load +// +// %1 = getelementptr <4 x i32>, ptr %vec4, i64 0 +// %2 = load <3 x i32>, ptr %1 +// +// The pointer in the GEP instruction is only used for offset computations, +// but it doesn't NEED to match the pointed type. OpAccessChain however +// requires this. Also, LLVM loads define the bitwidth of the load, not the +// pointer. In this example, we can guess %vec4 is a vec4 thanks to the GEP +// instruction basetype, but we only want to load the first 3 elements, hence +// do a partial load. In logical SPIR-V, this is not legal. What we must do +// is load the full vector (basetype), extract 3 elements, and recombine them +// to form a 3-element vector. +// +//===----------------------------------------------------------------------===// + +#include "SPIRV.h" +#include "SPIRVSubtarget.h" +#include "SPIRVTargetMachine.h" +#include "SPIRVUtils.h" +#include "llvm/CodeGen/IntrinsicLowering.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/LowerMemIntrinsics.h" + +using namespace llvm; + +namespace { +class SPIRVLegalizePointerCast : public FunctionPass { + + // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current + // builder position. + void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) { + Value *OfType = PoisonValue::get(Ty); + CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, + {Arg->getType()}, OfType, Arg, {}, B); + GR->addAssignPtrTypeInstr(Arg, AssignCI); + } + + // Loads parts of the vector of type |SourceType| from the pointer |Source| + // and create a new vector of type |TargetType|. |TargetType| must be a vector + // type, and element types of |TargetType| and |SourceType| must match. + // Returns the loaded value. + Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *SourceType, + FixedVectorType *TargetType, Value *Source) { + // We expect the codegen to avoid doing implicit bitcast from a load. + assert(TargetType->getElementType() == SourceType->getElementType()); + assert(TargetType->getNumElements() < SourceType->getNumElements()); + + LoadInst *NewLoad = B.CreateLoad(SourceType, Source); + buildAssignType(B, SourceType, NewLoad); + + SmallVector<int> Mask(/* Size= */ TargetType->getNumElements()); + for (unsigned I = 0; I < TargetType->getNumElements(); ++I) + Mask[I] = I; + Value *Output = B.CreateShuffleVector(NewLoad, NewLoad, Mask); + buildAssignType(B, TargetType, Output); + return Output; + } + + // Loads the first value in an aggregate pointed by |Source| of containing + // elements of type |ElementType|. Load flags will be copied from |BadLoad|, + // which should be the load being legalized. Returns the loaded value. + Value *loadFirstValueFromAggregate(IRBuilder<> &B, Type *ElementType, + Value *Source, LoadInst *BadLoad) { + SmallVector<Type *, 2> Types = {BadLoad->getPointerOperandType(), + BadLoad->getPointerOperandType()}; + SmallVector<Value *, 3> Args{/* isInBounds= */ B.getInt1(false), Source, + B.getInt32(0), B.getInt32(0)}; + auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); + GR->buildAssignPtr(B, ElementType, GEP); + + LoadInst *LI = B.CreateLoad(ElementType, GEP); + LI->setAlignment(BadLoad->getAlign()); + buildAssignType(B, ElementType, LI); + return LI; + } + + // Replaces the load instruction to get rid of the ptrcast used as source + // operand. + void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand, + Value *OriginalOperand) { + Type *FromTy = GR->findDeducedElementType(OriginalOperand); + Type *ToTy = GR->findDeducedElementType(CastedOperand); + Value *Output = nullptr; + + auto *SAT = dyn_cast<ArrayType>(FromTy); + auto *SVT = dyn_cast<FixedVectorType>(FromTy); + auto *SST = dyn_cast<StructType>(FromTy); + auto *DVT = dyn_cast<FixedVectorType>(ToTy); + + B.SetInsertPoint(LI); + + // Destination is the element type of Source, and source is an array -> + // Loading 1st element. + // - float a = array[0]; + if (SAT && SAT->getElementType() == ToTy) + Output = loadFirstValueFromAggregate(B, SAT->getElementType(), + OriginalOperand, LI); + // Destination is the element type of Source, and source is a vector -> + // Vector to scalar. + // - float a = vector.x; + else if (!DVT && SVT && SVT->getElementType() == ToTy) { + Output = loadFirstValueFromAggregate(B, SVT->getElementType(), + OriginalOperand, LI); + } + // Destination is a smaller vector than source. + // - float3 v3 = vector4; + else if (SVT && DVT) + Output = loadVectorFromVector(B, SVT, DVT, OriginalOperand); + // Destination is the scalar type stored at the start of an aggregate. + // - struct S { float m }; + // - float v = s.m; + else if (SST && SST->getTypeAtIndex(0u) == ToTy) + Output = loadFirstValueFromAggregate(B, ToTy, OriginalOperand, LI); + else + llvm_unreachable("Unimplemented implicit down-cast from load."); + + GR->replaceAllUsesWith(LI, Output, /* DeleteOld= */ true); + DeadInstructions.push_back(LI); + } + + // Creates an spv_insertelt instruction (equivalent to llvm's insertelement). + Value *makeInsertElement(IRBuilder<> &B, Value *Vector, Value *Element, + unsigned Index) { + Type *Int32Ty = Type::getInt32Ty(B.getContext()); + SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(), + Element->getType(), Int32Ty}; + SmallVector<Value *> Args = {Vector, Element, B.getInt32(Index)}; + Instruction *NewI = + B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); + buildAssignType(B, Vector->getType(), NewI); + return NewI; + } + + // Creates an spv_extractelt instruction (equivalent to llvm's + // extractelement). + Value *makeExtractElement(IRBuilder<> &B, Type *ElementType, Value *Vector, + unsigned Index) { + Type *Int32Ty = Type::getInt32Ty(B.getContext()); + SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty}; + SmallVector<Value *> Args = {Vector, B.getInt32(Index)}; + Instruction *NewI = + B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); + buildAssignType(B, ElementType, NewI); + return NewI; + } + + // Stores the given Src vector operand into the Dst vector, adjusting the size + // if required. + Value *storeVectorFromVector(IRBuilder<> &B, Value *Src, Value *Dst, + Align Alignment) { + FixedVectorType *SrcType = cast<FixedVectorType>(Src->getType()); + FixedVectorType *DstType = + cast<FixedVectorType>(GR->findDeducedElementType(Dst)); + assert(DstType->getNumElements() >= SrcType->getNumElements()); + + LoadInst *LI = B.CreateLoad(DstType, Dst); + LI->setAlignment(Alignment); + Value *OldValues = LI; + buildAssignType(B, OldValues->getType(), OldValues); + Value *NewValues = Src; + + for (unsigned I = 0; I < SrcType->getNumElements(); ++I) { + Value *Element = + makeExtractElement(B, SrcType->getElementType(), NewValues, I); + OldValues = makeInsertElement(B, OldValues, Element, I); + } + + StoreInst *SI = B.CreateStore(OldValues, Dst); + SI->setAlignment(Alignment); + return SI; + } + + void buildGEPIndexChain(IRBuilder<> &B, Type *Search, Type *Aggregate, + SmallVectorImpl<Value *> &Indices) { + Indices.push_back(B.getInt32(0)); + + if (Search == Aggregate) + return; + + if (auto *ST = dyn_cast<StructType>(Aggregate)) + buildGEPIndexChain(B, Search, ST->getTypeAtIndex(0u), Indices); + else if (auto *AT = dyn_cast<ArrayType>(Aggregate)) + buildGEPIndexChain(B, Search, AT->getElementType(), Indices); + else if (auto *VT = dyn_cast<FixedVectorType>(Aggregate)) + buildGEPIndexChain(B, Search, VT->getElementType(), Indices); + else + llvm_unreachable("Bad access chain?"); + } + + // Stores the given Src value into the first entry of the Dst aggregate. + Value *storeToFirstValueAggregate(IRBuilder<> &B, Value *Src, Value *Dst, + Type *DstPointeeType, Align Alignment) { + SmallVector<Type *, 2> Types = {Dst->getType(), Dst->getType()}; + SmallVector<Value *, 3> Args{/* isInBounds= */ B.getInt1(true), Dst}; + buildGEPIndexChain(B, Src->getType(), DstPointeeType, Args); + auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); + GR->buildAssignPtr(B, Src->getType(), GEP); + StoreInst *SI = B.CreateStore(Src, GEP); + SI->setAlignment(Alignment); + return SI; + } + + bool isTypeFirstElementAggregate(Type *Search, Type *Aggregate) { + if (Search == Aggregate) + return true; + if (auto *ST = dyn_cast<StructType>(Aggregate)) + return isTypeFirstElementAggregate(Search, ST->getTypeAtIndex(0u)); + if (auto *VT = dyn_cast<FixedVectorType>(Aggregate)) + return isTypeFirstElementAggregate(Search, VT->getElementType()); + if (auto *AT = dyn_cast<ArrayType>(Aggregate)) + return isTypeFirstElementAggregate(Search, AT->getElementType()); + return false; + } + + // Transforms a store instruction (or SPV intrinsic) using a ptrcast as + // operand into a valid logical SPIR-V store with no ptrcast. + void transformStore(IRBuilder<> &B, Instruction *BadStore, Value *Src, + Value *Dst, Align Alignment) { + Type *ToTy = GR->findDeducedElementType(Dst); + Type *FromTy = Src->getType(); + + auto *S_VT = dyn_cast<FixedVectorType>(FromTy); + auto *D_ST = dyn_cast<StructType>(ToTy); + auto *D_VT = dyn_cast<FixedVectorType>(ToTy); + + B.SetInsertPoint(BadStore); + if (D_ST && isTypeFirstElementAggregate(FromTy, D_ST)) + storeToFirstValueAggregate(B, Src, Dst, D_ST, Alignment); + else if (D_VT && S_VT) + storeVectorFromVector(B, Src, Dst, Alignment); + else if (D_VT && !S_VT && FromTy == D_VT->getElementType()) + storeToFirstValueAggregate(B, Src, Dst, D_VT, Alignment); + else + llvm_unreachable("Unsupported ptrcast use in store. Please fix."); + + DeadInstructions.push_back(BadStore); + } + + void legalizePointerCast(IntrinsicInst *II) { + Value *CastedOperand = II; + Value *OriginalOperand = II->getOperand(0); + + IRBuilder<> B(II->getContext()); + std::vector<Value *> Users; + for (Use &U : II->uses()) + Users.push_back(U.getUser()); + + for (Value *User : Users) { + if (LoadInst *LI = dyn_cast<LoadInst>(User)) { + transformLoad(B, LI, CastedOperand, OriginalOperand); + continue; + } + + if (StoreInst *SI = dyn_cast<StoreInst>(User)) { + transformStore(B, SI, SI->getValueOperand(), OriginalOperand, + SI->getAlign()); + continue; + } + + if (IntrinsicInst *Intrin = dyn_cast<IntrinsicInst>(User)) { + if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) { + DeadInstructions.push_back(Intrin); + continue; + } + + if (Intrin->getIntrinsicID() == Intrinsic::spv_gep) { + GR->replaceAllUsesWith(CastedOperand, OriginalOperand, + /* DeleteOld= */ false); + continue; + } + + if (Intrin->getIntrinsicID() == Intrinsic::spv_store) { + Align Alignment; + if (ConstantInt *C = dyn_cast<ConstantInt>(Intrin->getOperand(3))) + Alignment = Align(C->getZExtValue()); + transformStore(B, Intrin, Intrin->getArgOperand(0), OriginalOperand, + Alignment); + continue; + } + } + + llvm_unreachable("Unsupported ptrcast user. Please fix."); + } + + DeadInstructions.push_back(II); + } + +public: + SPIRVLegalizePointerCast(SPIRVTargetMachine *TM) : FunctionPass(ID), TM(TM) {} + + virtual bool runOnFunction(Function &F) override { + const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F); + GR = ST.getSPIRVGlobalRegistry(); + DeadInstructions.clear(); + + std::vector<IntrinsicInst *> WorkList; + for (auto &BB : F) { + for (auto &I : BB) { + auto *II = dyn_cast<IntrinsicInst>(&I); + if (II && II->getIntrinsicID() == Intrinsic::spv_ptrcast) + WorkList.push_back(II); + } + } + + for (IntrinsicInst *II : WorkList) + legalizePointerCast(II); + + for (Instruction *I : DeadInstructions) + I->eraseFromParent(); + + return DeadInstructions.size() != 0; + } + +private: + SPIRVTargetMachine *TM = nullptr; + SPIRVGlobalRegistry *GR = nullptr; + std::vector<Instruction *> DeadInstructions; + +public: + static char ID; +}; +} // namespace + +char SPIRVLegalizePointerCast::ID = 0; +INITIALIZE_PASS(SPIRVLegalizePointerCast, "spirv-legalize-bitcast", + "SPIRV legalize bitcast pass", false, false) + +FunctionPass *llvm::createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM) { + return new SPIRVLegalizePointerCast(TM); +} |