diff options
Diffstat (limited to 'include/llvm/Transforms/Vectorize')
4 files changed, 82 insertions, 52 deletions
diff --git a/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h b/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h index 6b37d7093c44..f72c76c6f0f2 100644 --- a/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h +++ b/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h @@ -1,9 +1,8 @@ //===- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index 5c7bba048607..b144006e2628 100644 --- a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Vectorize/LoopVectorizationLegality.h ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -99,11 +98,7 @@ public: OptimizationRemarkEmitter &ORE); /// Mark the loop L as already vectorized by setting the width to 1. - void setAlreadyVectorized() { - IsVectorized.Value = 1; - Hint Hints[] = {IsVectorized}; - writeHintsToMetadata(Hints); - } + void setAlreadyVectorized(); bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const; @@ -152,15 +147,6 @@ private: /// Checks string hint with one operand and set value if valid. void setHint(StringRef Name, Metadata *Arg); - /// Create a new hint from name / value pair. - MDNode *createHintMetadata(StringRef Name, unsigned V) const; - - /// Matches metadata with hint name. - bool matchesHintMetadataName(MDNode *Node, ArrayRef<Hint> HintTypes); - - /// Sets current hints into loop metadata, keeping other values intact. - void writeHintsToMetadata(ArrayRef<Hint> HintTypes); - /// The loop these hints belong to. const Loop *TheLoop; @@ -219,12 +205,13 @@ class LoopVectorizationLegality { public: LoopVectorizationLegality( Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT, - TargetLibraryInfo *TLI, AliasAnalysis *AA, Function *F, - std::function<const LoopAccessInfo &(Loop &)> *GetLAA, LoopInfo *LI, - OptimizationRemarkEmitter *ORE, LoopVectorizationRequirements *R, - LoopVectorizeHints *H, DemandedBits *DB, AssumptionCache *AC) - : TheLoop(L), LI(LI), PSE(PSE), TLI(TLI), DT(DT), GetLAA(GetLAA), - ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC) {} + TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AliasAnalysis *AA, + Function *F, std::function<const LoopAccessInfo &(Loop &)> *GetLAA, + LoopInfo *LI, OptimizationRemarkEmitter *ORE, + LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB, + AssumptionCache *AC) + : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT), + GetLAA(GetLAA), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC) {} /// ReductionList contains the reduction descriptors for all /// of the reductions that were found in the loop. @@ -385,18 +372,6 @@ private: void addInductionPhi(PHINode *Phi, const InductionDescriptor &ID, SmallPtrSetImpl<Value *> &AllowedExit); - /// Create an analysis remark that explains why vectorization failed - /// - /// \p RemarkName is the identifier for the remark. If \p I is passed it is - /// an instruction that prevents vectorization. Otherwise the loop is used - /// for the location of the remark. \return the remark object that can be - /// streamed to. - OptimizationRemarkAnalysis - createMissedAnalysis(StringRef RemarkName, Instruction *I = nullptr) const { - return createLVMissedAnalysis(Hints->vectorizeAnalysisPassName(), - RemarkName, TheLoop, I); - } - /// If an access has a symbolic strides, this maps the pointer value to /// the stride symbol. const ValueToValueMap *getSymbolicStrides() { @@ -407,6 +382,14 @@ private: return LAI ? &LAI->getSymbolicStrides() : nullptr; } + /// Reports a vectorization illegality: print \p DebugMsg for debugging + /// purposes along with the corresponding optimization remark \p RemarkName. + /// If \p I is passed it is an instruction that prevents vectorization. + /// Otherwise the loop is used for the location of the remark. + void reportVectorizationFailure(const StringRef DebugMsg, + const StringRef OREMsg, const StringRef ORETag, + Instruction *I = nullptr) const; + /// The loop that we evaluate. Loop *TheLoop; @@ -420,6 +403,9 @@ private: /// unrolling. PredicatedScalarEvolution &PSE; + /// Target Transform Info. + TargetTransformInfo *TTI; + /// Target Library Info. TargetLibraryInfo *TLI; @@ -479,7 +465,7 @@ private: /// Used to emit an analysis of any legality issues. LoopVectorizeHints *Hints; - /// The demanded bits analsyis is used to compute the minimum type size in + /// The demanded bits analysis is used to compute the minimum type size in /// which a reduction can be computed. DemandedBits *DB; diff --git a/include/llvm/Transforms/Vectorize/LoopVectorize.h b/include/llvm/Transforms/Vectorize/LoopVectorize.h index d9c4f7b023c1..d1ec06afb02a 100644 --- a/include/llvm/Transforms/Vectorize/LoopVectorize.h +++ b/include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -1,9 +1,8 @@ //===- LoopVectorize.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -72,19 +71,63 @@ class Loop; class LoopAccessInfo; class LoopInfo; class OptimizationRemarkEmitter; +class ProfileSummaryInfo; class ScalarEvolution; class TargetLibraryInfo; class TargetTransformInfo; +extern cl::opt<bool> EnableLoopInterleaving; +extern cl::opt<bool> EnableLoopVectorization; + +struct LoopVectorizeOptions { + /// If false, consider all loops for interleaving. + /// If true, only loops that explicitly request interleaving are considered. + bool InterleaveOnlyWhenForced; + + /// If false, consider all loops for vectorization. + /// If true, only loops that explicitly request vectorization are considered. + bool VectorizeOnlyWhenForced; + + /// The current defaults when creating the pass with no arguments are: + /// EnableLoopInterleaving = true and EnableLoopVectorization = true. This + /// means that interleaving default is consistent with the cl::opt flag, while + /// vectorization is not. + /// FIXME: The default for EnableLoopVectorization in the cl::opt should be + /// set to true, and the corresponding change to account for this be made in + /// opt.cpp. The initializations below will become: + /// InterleaveOnlyWhenForced(!EnableLoopInterleaving) + /// VectorizeOnlyWhenForced(!EnableLoopVectorization). + LoopVectorizeOptions() + : InterleaveOnlyWhenForced(false), VectorizeOnlyWhenForced(false) {} + LoopVectorizeOptions(bool InterleaveOnlyWhenForced, + bool VectorizeOnlyWhenForced) + : InterleaveOnlyWhenForced(InterleaveOnlyWhenForced), + VectorizeOnlyWhenForced(VectorizeOnlyWhenForced) {} + + LoopVectorizeOptions &setInterleaveOnlyWhenForced(bool Value) { + InterleaveOnlyWhenForced = Value; + return *this; + } + + LoopVectorizeOptions &setVectorizeOnlyWhenForced(bool Value) { + VectorizeOnlyWhenForced = Value; + return *this; + } +}; + /// The LoopVectorize Pass. struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> { /// If false, consider all loops for interleaving. /// If true, only loops that explicitly request interleaving are considered. - bool InterleaveOnlyWhenForced = false; + bool InterleaveOnlyWhenForced; /// If false, consider all loops for vectorization. /// If true, only loops that explicitly request vectorization are considered. - bool VectorizeOnlyWhenForced = false; + bool VectorizeOnlyWhenForced; + + LoopVectorizePass(LoopVectorizeOptions Opts = {}) + : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced), + VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {} ScalarEvolution *SE; LoopInfo *LI; @@ -97,6 +140,7 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> { AssumptionCache *AC; std::function<const LoopAccessInfo &(Loop &)> *GetLAA; OptimizationRemarkEmitter *ORE; + ProfileSummaryInfo *PSI; PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); @@ -106,7 +150,7 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> { BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_, DemandedBits &DB_, AliasAnalysis &AA_, AssumptionCache &AC_, std::function<const LoopAccessInfo &(Loop &)> &GetLAA_, - OptimizationRemarkEmitter &ORE); + OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_); bool processLoop(Loop *L); }; diff --git a/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/include/llvm/Transforms/Vectorize/SLPVectorizer.h index 3152e8192fc5..ac6afb761d4d 100644 --- a/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -1,9 +1,8 @@ //===- SLPVectorizer.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This pass implements the Bottom Up SLP vectorizer. It detects consecutive @@ -56,6 +55,8 @@ class BoUpSLP; } // end namespace slpvectorizer +extern cl::opt<bool> RunSLPVectorization; + struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> { using StoreList = SmallVector<StoreInst *, 8>; using StoreListMap = MapVector<Value *, StoreList>; |