diff options
Diffstat (limited to 'llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp')
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp | 108 |
1 files changed, 55 insertions, 53 deletions
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp index 3b90997100f1..5a4a2f0924f6 100644 --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -694,31 +694,16 @@ Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) { }); for (Instruction &I : make_range(getBoundaryInstrs(Chain))) { - if (isa<LoadInst>(I) || isa<StoreInst>(I)) { - if (!is_contained(Chain, &I)) - MemoryInstrs.push_back(&I); - else - ChainInstrs.push_back(&I); - } else if (isa<IntrinsicInst>(&I) && - cast<IntrinsicInst>(&I)->getIntrinsicID() == - Intrinsic::sideeffect) { - // Ignore llvm.sideeffect calls. - } else if (isa<IntrinsicInst>(&I) && - cast<IntrinsicInst>(&I)->getIntrinsicID() == - Intrinsic::pseudoprobe) { - // Ignore llvm.pseudoprobe calls. - } else if (isa<IntrinsicInst>(&I) && - cast<IntrinsicInst>(&I)->getIntrinsicID() == Intrinsic::assume) { - // Ignore llvm.assume calls. - } else if (IsLoadChain && (I.mayWriteToMemory() || I.mayThrow())) { - LLVM_DEBUG(dbgs() << "LSV: Found may-write/throw operation: " << I - << '\n'); - break; - } else if (!IsLoadChain && (I.mayReadOrWriteMemory() || I.mayThrow())) { - LLVM_DEBUG(dbgs() << "LSV: Found may-read/write/throw operation: " << I - << '\n'); + if ((isa<LoadInst>(I) || isa<StoreInst>(I)) && is_contained(Chain, &I)) { + ChainInstrs.push_back(&I); + continue; + } + if (I.mayThrow()) { + LLVM_DEBUG(dbgs() << "LSV: Found may-throw operation: " << I << '\n'); break; } + if (I.mayReadOrWriteMemory()) + MemoryInstrs.push_back(&I); } // Loop until we find an instruction in ChainInstrs that we can't vectorize. @@ -751,26 +736,28 @@ Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) { return LI->hasMetadata(LLVMContext::MD_invariant_load); }; - // We can ignore the alias as long as the load comes before the store, - // because that means we won't be moving the load past the store to - // vectorize it (the vectorized load is inserted at the location of the - // first load in the chain). - if (isa<StoreInst>(MemInstr) && ChainLoad && - (IsInvariantLoad(ChainLoad) || ChainLoad->comesBefore(MemInstr))) - continue; - - // Same case, but in reverse. - if (MemLoad && isa<StoreInst>(ChainInstr) && - (IsInvariantLoad(MemLoad) || MemLoad->comesBefore(ChainInstr))) - continue; + if (IsLoadChain) { + // We can ignore the alias as long as the load comes before the store, + // because that means we won't be moving the load past the store to + // vectorize it (the vectorized load is inserted at the location of the + // first load in the chain). + if (ChainInstr->comesBefore(MemInstr) || + (ChainLoad && IsInvariantLoad(ChainLoad))) + continue; + } else { + // Same case, but in reverse. + if (MemInstr->comesBefore(ChainInstr) || + (MemLoad && IsInvariantLoad(MemLoad))) + continue; + } - if (!AA.isNoAlias(MemoryLocation::get(MemInstr), - MemoryLocation::get(ChainInstr))) { + ModRefInfo MR = + AA.getModRefInfo(MemInstr, MemoryLocation::get(ChainInstr)); + if (IsLoadChain ? isModSet(MR) : isModOrRefSet(MR)) { LLVM_DEBUG({ dbgs() << "LSV: Found alias:\n" - " Aliasing instruction and pointer:\n" + " Aliasing instruction:\n" << " " << *MemInstr << '\n' - << " " << *getLoadStorePointerOperand(MemInstr) << '\n' << " Aliased instruction and pointer:\n" << " " << *ChainInstr << '\n' << " " << *getLoadStorePointerOperand(ChainInstr) << '\n'; @@ -1085,9 +1072,12 @@ bool Vectorizer::vectorizeStoreChain( if (ChainSize > VF || (VF != TargetVF && TargetVF < ChainSize)) { LLVM_DEBUG(dbgs() << "LSV: Chain doesn't match with the vector factor." " Creating two separate arrays.\n"); - return vectorizeStoreChain(Chain.slice(0, TargetVF), - InstructionsProcessed) | - vectorizeStoreChain(Chain.slice(TargetVF), InstructionsProcessed); + bool Vectorized = false; + Vectorized |= + vectorizeStoreChain(Chain.slice(0, TargetVF), InstructionsProcessed); + Vectorized |= + vectorizeStoreChain(Chain.slice(TargetVF), InstructionsProcessed); + return Vectorized; } LLVM_DEBUG({ @@ -1104,8 +1094,10 @@ bool Vectorizer::vectorizeStoreChain( if (accessIsMisaligned(SzInBytes, AS, Alignment)) { if (S0->getPointerAddressSpace() != DL.getAllocaAddrSpace()) { auto Chains = splitOddVectorElts(Chain, Sz); - return vectorizeStoreChain(Chains.first, InstructionsProcessed) | - vectorizeStoreChain(Chains.second, InstructionsProcessed); + bool Vectorized = false; + Vectorized |= vectorizeStoreChain(Chains.first, InstructionsProcessed); + Vectorized |= vectorizeStoreChain(Chains.second, InstructionsProcessed); + return Vectorized; } Align NewAlign = getOrEnforceKnownAlignment(S0->getPointerOperand(), @@ -1119,15 +1111,17 @@ bool Vectorizer::vectorizeStoreChain( if (!TTI.isLegalToVectorizeStoreChain(SzInBytes, Alignment, AS)) { auto Chains = splitOddVectorElts(Chain, Sz); - return vectorizeStoreChain(Chains.first, InstructionsProcessed) | - vectorizeStoreChain(Chains.second, InstructionsProcessed); + bool Vectorized = false; + Vectorized |= vectorizeStoreChain(Chains.first, InstructionsProcessed); + Vectorized |= vectorizeStoreChain(Chains.second, InstructionsProcessed); + return Vectorized; } BasicBlock::iterator First, Last; std::tie(First, Last) = getBoundaryInstrs(Chain); Builder.SetInsertPoint(&*Last); - Value *Vec = UndefValue::get(VecTy); + Value *Vec = PoisonValue::get(VecTy); if (VecStoreTy) { unsigned VecWidth = VecStoreTy->getNumElements(); @@ -1237,8 +1231,12 @@ bool Vectorizer::vectorizeLoadChain( if (ChainSize > VF || (VF != TargetVF && TargetVF < ChainSize)) { LLVM_DEBUG(dbgs() << "LSV: Chain doesn't match with the vector factor." " Creating two separate arrays.\n"); - return vectorizeLoadChain(Chain.slice(0, TargetVF), InstructionsProcessed) | - vectorizeLoadChain(Chain.slice(TargetVF), InstructionsProcessed); + bool Vectorized = false; + Vectorized |= + vectorizeLoadChain(Chain.slice(0, TargetVF), InstructionsProcessed); + Vectorized |= + vectorizeLoadChain(Chain.slice(TargetVF), InstructionsProcessed); + return Vectorized; } // We won't try again to vectorize the elements of the chain, regardless of @@ -1249,8 +1247,10 @@ bool Vectorizer::vectorizeLoadChain( if (accessIsMisaligned(SzInBytes, AS, Alignment)) { if (L0->getPointerAddressSpace() != DL.getAllocaAddrSpace()) { auto Chains = splitOddVectorElts(Chain, Sz); - return vectorizeLoadChain(Chains.first, InstructionsProcessed) | - vectorizeLoadChain(Chains.second, InstructionsProcessed); + bool Vectorized = false; + Vectorized |= vectorizeLoadChain(Chains.first, InstructionsProcessed); + Vectorized |= vectorizeLoadChain(Chains.second, InstructionsProcessed); + return Vectorized; } Align NewAlign = getOrEnforceKnownAlignment(L0->getPointerOperand(), @@ -1264,8 +1264,10 @@ bool Vectorizer::vectorizeLoadChain( if (!TTI.isLegalToVectorizeLoadChain(SzInBytes, Alignment, AS)) { auto Chains = splitOddVectorElts(Chain, Sz); - return vectorizeLoadChain(Chains.first, InstructionsProcessed) | - vectorizeLoadChain(Chains.second, InstructionsProcessed); + bool Vectorized = false; + Vectorized |= vectorizeLoadChain(Chains.first, InstructionsProcessed); + Vectorized |= vectorizeLoadChain(Chains.second, InstructionsProcessed); + return Vectorized; } LLVM_DEBUG({ |