diff options
Diffstat (limited to 'llvm/lib/Analysis')
55 files changed, 4225 insertions, 2381 deletions
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp index e7445e225d52..d030f74481cf 100644 --- a/llvm/lib/Analysis/AliasAnalysis.cpp +++ b/llvm/lib/Analysis/AliasAnalysis.cpp @@ -119,7 +119,7 @@ bool AAResults::invalidate(Function &F, const PreservedAnalyses &PA, AliasResult AAResults::alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return alias(LocA, LocB, AAQIP); } @@ -162,7 +162,7 @@ AliasResult AAResults::alias(const MemoryLocation &LocA, bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return pointsToConstantMemory(Loc, AAQIP, OrLocal); } @@ -190,7 +190,7 @@ ModRefInfo AAResults::getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { } ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(I, Call2, AAQIP); } @@ -200,25 +200,24 @@ ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2, if (const auto *Call1 = dyn_cast<CallBase>(I)) { // Check if the two calls modify the same memory. return getModRefInfo(Call1, Call2, AAQI); - } else if (I->isFenceLike()) { - // If this is a fence, just return ModRef. - return ModRefInfo::ModRef; - } else { - // Otherwise, check if the call modifies or references the - // location this memory access defines. The best we can say - // is that if the call references what this instruction - // defines, it must be clobbered by this location. - const MemoryLocation DefLoc = MemoryLocation::get(I); - ModRefInfo MR = getModRefInfo(Call2, DefLoc, AAQI); - if (isModOrRefSet(MR)) - return setModAndRef(MR); } + // If this is a fence, just return ModRef. + if (I->isFenceLike()) + return ModRefInfo::ModRef; + // Otherwise, check if the call modifies or references the + // location this memory access defines. The best we can say + // is that if the call references what this instruction + // defines, it must be clobbered by this location. + const MemoryLocation DefLoc = MemoryLocation::get(I); + ModRefInfo MR = getModRefInfo(Call2, DefLoc, AAQI); + if (isModOrRefSet(MR)) + return setModAndRef(MR); return ModRefInfo::NoModRef; } ModRefInfo AAResults::getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(Call, Loc, AAQIP); } @@ -285,7 +284,7 @@ ModRefInfo AAResults::getModRefInfo(const CallBase *Call, ModRefInfo AAResults::getModRefInfo(const CallBase *Call1, const CallBase *Call2) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(Call1, Call2, AAQIP); } @@ -475,7 +474,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, AliasResult AR) { ModRefInfo AAResults::getModRefInfo(const LoadInst *L, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(L, Loc, AAQIP); } ModRefInfo AAResults::getModRefInfo(const LoadInst *L, @@ -500,7 +499,7 @@ ModRefInfo AAResults::getModRefInfo(const LoadInst *L, ModRefInfo AAResults::getModRefInfo(const StoreInst *S, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(S, Loc, AAQIP); } ModRefInfo AAResults::getModRefInfo(const StoreInst *S, @@ -532,7 +531,7 @@ ModRefInfo AAResults::getModRefInfo(const StoreInst *S, } ModRefInfo AAResults::getModRefInfo(const FenceInst *S, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(S, Loc, AAQIP); } @@ -548,7 +547,7 @@ ModRefInfo AAResults::getModRefInfo(const FenceInst *S, ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(V, Loc, AAQIP); } @@ -578,7 +577,7 @@ ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, ModRefInfo AAResults::getModRefInfo(const CatchPadInst *CatchPad, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(CatchPad, Loc, AAQIP); } @@ -598,7 +597,7 @@ ModRefInfo AAResults::getModRefInfo(const CatchPadInst *CatchPad, ModRefInfo AAResults::getModRefInfo(const CatchReturnInst *CatchRet, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(CatchRet, Loc, AAQIP); } @@ -618,7 +617,7 @@ ModRefInfo AAResults::getModRefInfo(const CatchReturnInst *CatchRet, ModRefInfo AAResults::getModRefInfo(const AtomicCmpXchgInst *CX, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(CX, Loc, AAQIP); } @@ -646,7 +645,7 @@ ModRefInfo AAResults::getModRefInfo(const AtomicCmpXchgInst *CX, ModRefInfo AAResults::getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc) { - AAQueryInfo AAQIP; + SimpleAAQueryInfo AAQIP; return getModRefInfo(RMW, Loc, AAQIP); } @@ -746,7 +745,7 @@ ModRefInfo AAResults::callCapturesBefore(const Instruction *I, // pointer were passed to arguments that were neither of these, then it // couldn't be no-capture. if (!(*CI)->getType()->isPointerTy() || - (!Call->doesNotCapture(ArgNo) && ArgNo < Call->getNumArgOperands() && + (!Call->doesNotCapture(ArgNo) && ArgNo < Call->arg_size() && !Call->isByValArgument(ArgNo))) continue; @@ -808,11 +807,6 @@ AAResults::Concept::~Concept() = default; // Provide a definition for the static object used to identify passes. AnalysisKey AAManager::Key; -namespace { - - -} // end anonymous namespace - ExternalAAWrapperPass::ExternalAAWrapperPass() : ImmutablePass(ID) { initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry()); } diff --git a/llvm/lib/Analysis/AssumeBundleQueries.cpp b/llvm/lib/Analysis/AssumeBundleQueries.cpp index dee044346f02..9d4fe1225b33 100644 --- a/llvm/lib/Analysis/AssumeBundleQueries.cpp +++ b/llvm/lib/Analysis/AssumeBundleQueries.cpp @@ -84,7 +84,7 @@ void llvm::fillMapFromAssume(AssumeInst &Assume, RetainedKnowledgeMap &Result) { getValueFromBundleOpInfo(Assume, Bundles, ABA_Argument)); if (!CI) continue; - unsigned Val = CI->getZExtValue(); + uint64_t Val = CI->getZExtValue(); auto Lookup = Result.find(Key); if (Lookup == Result.end() || !Lookup->second.count(&Assume)) { Result[Key][&Assume] = {Val, Val}; @@ -102,7 +102,7 @@ llvm::getKnowledgeFromBundle(AssumeInst &Assume, Result.AttrKind = Attribute::getAttrKindFromName(BOI.Tag->getKey()); if (bundleHasArgument(BOI, ABA_WasOn)) Result.WasOn = getValueFromBundleOpInfo(Assume, BOI, ABA_WasOn); - auto GetArgOr1 = [&](unsigned Idx) -> unsigned { + auto GetArgOr1 = [&](unsigned Idx) -> uint64_t { if (auto *ConstInt = dyn_cast<ConstantInt>( getValueFromBundleOpInfo(Assume, BOI, ABA_Argument + Idx))) return ConstInt->getZExtValue(); diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp index 0d95b33601f9..3e0214e21ecd 100644 --- a/llvm/lib/Analysis/AssumptionCache.cpp +++ b/llvm/lib/Analysis/AssumptionCache.cpp @@ -16,6 +16,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" @@ -56,7 +57,7 @@ AssumptionCache::getOrInsertAffectedValues(Value *V) { } static void -findAffectedValues(CallBase *CI, +findAffectedValues(CallBase *CI, TargetTransformInfo *TTI, SmallVectorImpl<AssumptionCache::ResultElem> &Affected) { // Note: This code must be kept in-sync with the code in // computeKnownBitsFromAssume in ValueTracking. @@ -124,24 +125,32 @@ findAffectedValues(CallBase *CI, match(B, m_ConstantInt())) AddAffected(X); } + + if (TTI) { + const Value *Ptr; + unsigned AS; + std::tie(Ptr, AS) = TTI->getPredicatedAddrSpace(Cond); + if (Ptr) + AddAffected(const_cast<Value *>(Ptr->stripInBoundsOffsets())); + } } void AssumptionCache::updateAffectedValues(AssumeInst *CI) { SmallVector<AssumptionCache::ResultElem, 16> Affected; - findAffectedValues(CI, Affected); + findAffectedValues(CI, TTI, Affected); for (auto &AV : Affected) { auto &AVV = getOrInsertAffectedValues(AV.Assume); - if (std::find_if(AVV.begin(), AVV.end(), [&](ResultElem &Elem) { + if (llvm::none_of(AVV, [&](ResultElem &Elem) { return Elem.Assume == CI && Elem.Index == AV.Index; - }) == AVV.end()) + })) AVV.push_back({CI, AV.Index}); } } void AssumptionCache::unregisterAssumption(AssumeInst *CI) { SmallVector<AssumptionCache::ResultElem, 16> Affected; - findAffectedValues(CI, Affected); + findAffectedValues(CI, TTI, Affected); for (auto &AV : Affected) { auto AVI = AffectedValues.find_as(AV.Assume); @@ -248,6 +257,12 @@ void AssumptionCache::registerAssumption(AssumeInst *CI) { updateAffectedValues(CI); } +AssumptionCache AssumptionAnalysis::run(Function &F, + FunctionAnalysisManager &FAM) { + auto &TTI = FAM.getResult<TargetIRAnalysis>(F); + return AssumptionCache(F, &TTI); +} + AnalysisKey AssumptionAnalysis::Key; PreservedAnalyses AssumptionPrinterPass::run(Function &F, @@ -278,10 +293,13 @@ AssumptionCache &AssumptionCacheTracker::getAssumptionCache(Function &F) { if (I != AssumptionCaches.end()) return *I->second; + auto *TTIWP = getAnalysisIfAvailable<TargetTransformInfoWrapperPass>(); + auto *TTI = TTIWP ? &TTIWP->getTTI(F) : nullptr; + // Ok, build a new cache by scanning the function, insert it and the value // handle into our map, and return the newly populated cache. auto IP = AssumptionCaches.insert(std::make_pair( - FunctionCallbackVH(&F, this), std::make_unique<AssumptionCache>(F))); + FunctionCallbackVH(&F, this), std::make_unique<AssumptionCache>(F, TTI))); assert(IP.second && "Scanning function already in the map?"); return *IP.first->second; } diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp index 357772c9c4f2..88b0f37b1d48 100644 --- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constant.h" +#include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" @@ -68,15 +69,6 @@ using namespace llvm; static cl::opt<bool> EnableRecPhiAnalysis("basic-aa-recphi", cl::Hidden, cl::init(true)); -/// By default, even on 32-bit architectures we use 64-bit integers for -/// calculations. This will allow us to more-aggressively decompose indexing -/// expressions calculated using i64 values (e.g., long long in C) which is -/// common enough to worry about. -static cl::opt<bool> ForceAtLeast64Bits("basic-aa-force-at-least-64b", - cl::Hidden, cl::init(true)); -static cl::opt<bool> DoubleCalcBits("basic-aa-double-calc-bits", - cl::Hidden, cl::init(false)); - /// SearchLimitReached / SearchTimes shows how often the limit of /// to decompose GEPs is reached. It will affect the precision /// of basic alias analysis. @@ -91,8 +83,7 @@ STATISTIC(SearchTimes, "Number of times a GEP is decomposed"); const unsigned MaxNumPhiBBsValueReachabilityCheck = 20; // The max limit of the search depth in DecomposeGEPExpression() and -// getUnderlyingObject(), both functions need to use the same search -// depth otherwise the algorithm in aliasGEP will assert. +// getUnderlyingObject(). static const unsigned MaxLookupSearchDepth = 6; bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA, @@ -120,9 +111,6 @@ static bool isEscapeSource(const Value *V) { if (isa<CallBase>(V)) return true; - if (isa<Argument>(V)) - return true; - // The load case works because isNonEscapingLocalObject considers all // stores to be escapes (it passes true for the StoreCaptures argument // to PointerMayBeCaptured). @@ -206,12 +194,12 @@ static uint64_t getMinimalExtentFrom(const Value &V, bool NullIsValidLoc) { // If we have dereferenceability information we know a lower bound for the // extent as accesses for a lower offset would be valid. We need to exclude - // the "or null" part if null is a valid pointer. + // the "or null" part if null is a valid pointer. We can ignore frees, as an + // access after free would be undefined behavior. bool CanBeNull, CanBeFreed; uint64_t DerefBytes = V.getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed); DerefBytes = (CanBeNull && NullIsValidLoc) ? 0 : DerefBytes; - DerefBytes = CanBeFreed ? 0 : DerefBytes; // If queried with a precise location size, we assume that location size to be // accessed, thus valid. if (LocSize.isPrecise()) @@ -227,82 +215,163 @@ static bool isObjectSize(const Value *V, uint64_t Size, const DataLayout &DL, } //===----------------------------------------------------------------------===// +// CaptureInfo implementations +//===----------------------------------------------------------------------===// + +CaptureInfo::~CaptureInfo() = default; + +bool SimpleCaptureInfo::isNotCapturedBeforeOrAt(const Value *Object, + const Instruction *I) { + return isNonEscapingLocalObject(Object, &IsCapturedCache); +} + +bool EarliestEscapeInfo::isNotCapturedBeforeOrAt(const Value *Object, + const Instruction *I) { + if (!isIdentifiedFunctionLocal(Object)) + return false; + + auto Iter = EarliestEscapes.insert({Object, nullptr}); + if (Iter.second) { + Instruction *EarliestCapture = FindEarliestCapture( + Object, *const_cast<Function *>(I->getFunction()), + /*ReturnCaptures=*/false, /*StoreCaptures=*/true, DT); + if (EarliestCapture) { + auto Ins = Inst2Obj.insert({EarliestCapture, {}}); + Ins.first->second.push_back(Object); + } + Iter.first->second = EarliestCapture; + } + + // No capturing instruction. + if (!Iter.first->second) + return true; + + return I != Iter.first->second && + !isPotentiallyReachable(Iter.first->second, I, nullptr, &DT, &LI); +} + +void EarliestEscapeInfo::removeInstruction(Instruction *I) { + auto Iter = Inst2Obj.find(I); + if (Iter != Inst2Obj.end()) { + for (const Value *Obj : Iter->second) + EarliestEscapes.erase(Obj); + Inst2Obj.erase(I); + } +} + +//===----------------------------------------------------------------------===// // GetElementPtr Instruction Decomposition and Analysis //===----------------------------------------------------------------------===// namespace { -/// Represents zext(sext(V)). -struct ExtendedValue { +/// Represents zext(sext(trunc(V))). +struct CastedValue { const Value *V; - unsigned ZExtBits; - unsigned SExtBits; + unsigned ZExtBits = 0; + unsigned SExtBits = 0; + unsigned TruncBits = 0; - explicit ExtendedValue(const Value *V, unsigned ZExtBits = 0, - unsigned SExtBits = 0) - : V(V), ZExtBits(ZExtBits), SExtBits(SExtBits) {} + explicit CastedValue(const Value *V) : V(V) {} + explicit CastedValue(const Value *V, unsigned ZExtBits, unsigned SExtBits, + unsigned TruncBits) + : V(V), ZExtBits(ZExtBits), SExtBits(SExtBits), TruncBits(TruncBits) {} unsigned getBitWidth() const { - return V->getType()->getPrimitiveSizeInBits() + ZExtBits + SExtBits; + return V->getType()->getPrimitiveSizeInBits() - TruncBits + ZExtBits + + SExtBits; } - ExtendedValue withValue(const Value *NewV) const { - return ExtendedValue(NewV, ZExtBits, SExtBits); + CastedValue withValue(const Value *NewV) const { + return CastedValue(NewV, ZExtBits, SExtBits, TruncBits); } - ExtendedValue withZExtOfValue(const Value *NewV) const { + /// Replace V with zext(NewV) + CastedValue withZExtOfValue(const Value *NewV) const { unsigned ExtendBy = V->getType()->getPrimitiveSizeInBits() - NewV->getType()->getPrimitiveSizeInBits(); + if (ExtendBy <= TruncBits) + return CastedValue(NewV, ZExtBits, SExtBits, TruncBits - ExtendBy); + // zext(sext(zext(NewV))) == zext(zext(zext(NewV))) - return ExtendedValue(NewV, ZExtBits + SExtBits + ExtendBy, 0); + ExtendBy -= TruncBits; + return CastedValue(NewV, ZExtBits + SExtBits + ExtendBy, 0, 0); } - ExtendedValue withSExtOfValue(const Value *NewV) const { + /// Replace V with sext(NewV) + CastedValue withSExtOfValue(const Value *NewV) const { unsigned ExtendBy = V->getType()->getPrimitiveSizeInBits() - NewV->getType()->getPrimitiveSizeInBits(); + if (ExtendBy <= TruncBits) + return CastedValue(NewV, ZExtBits, SExtBits, TruncBits - ExtendBy); + // zext(sext(sext(NewV))) - return ExtendedValue(NewV, ZExtBits, SExtBits + ExtendBy); + ExtendBy -= TruncBits; + return CastedValue(NewV, ZExtBits, SExtBits + ExtendBy, 0); } APInt evaluateWith(APInt N) const { assert(N.getBitWidth() == V->getType()->getPrimitiveSizeInBits() && "Incompatible bit width"); + if (TruncBits) N = N.trunc(N.getBitWidth() - TruncBits); if (SExtBits) N = N.sext(N.getBitWidth() + SExtBits); if (ZExtBits) N = N.zext(N.getBitWidth() + ZExtBits); return N; } + ConstantRange evaluateWith(ConstantRange N) const { + assert(N.getBitWidth() == V->getType()->getPrimitiveSizeInBits() && + "Incompatible bit width"); + if (TruncBits) N = N.truncate(N.getBitWidth() - TruncBits); + if (SExtBits) N = N.signExtend(N.getBitWidth() + SExtBits); + if (ZExtBits) N = N.zeroExtend(N.getBitWidth() + ZExtBits); + return N; + } + bool canDistributeOver(bool NUW, bool NSW) const { // zext(x op<nuw> y) == zext(x) op<nuw> zext(y) // sext(x op<nsw> y) == sext(x) op<nsw> sext(y) + // trunc(x op y) == trunc(x) op trunc(y) return (!ZExtBits || NUW) && (!SExtBits || NSW); } + + bool hasSameCastsAs(const CastedValue &Other) const { + return ZExtBits == Other.ZExtBits && SExtBits == Other.SExtBits && + TruncBits == Other.TruncBits; + } }; -/// Represents zext(sext(V)) * Scale + Offset. +/// Represents zext(sext(trunc(V))) * Scale + Offset. struct LinearExpression { - ExtendedValue Val; + CastedValue Val; APInt Scale; APInt Offset; /// True if all operations in this expression are NSW. bool IsNSW; - LinearExpression(const ExtendedValue &Val, const APInt &Scale, + LinearExpression(const CastedValue &Val, const APInt &Scale, const APInt &Offset, bool IsNSW) : Val(Val), Scale(Scale), Offset(Offset), IsNSW(IsNSW) {} - LinearExpression(const ExtendedValue &Val) : Val(Val), IsNSW(true) { + LinearExpression(const CastedValue &Val) : Val(Val), IsNSW(true) { unsigned BitWidth = Val.getBitWidth(); Scale = APInt(BitWidth, 1); Offset = APInt(BitWidth, 0); } + + LinearExpression mul(const APInt &Other, bool MulIsNSW) const { + // The check for zero offset is necessary, because generally + // (X +nsw Y) *nsw Z does not imply (X *nsw Z) +nsw (Y *nsw Z). + bool NSW = IsNSW && (Other.isOne() || (MulIsNSW && Offset.isZero())); + return LinearExpression(Val, Scale * Other, Offset * Other, NSW); + } }; } /// Analyzes the specified value as a linear expression: "A*V + B", where A and /// B are constant integers. static LinearExpression GetLinearExpression( - const ExtendedValue &Val, const DataLayout &DL, unsigned Depth, + const CastedValue &Val, const DataLayout &DL, unsigned Depth, AssumptionCache *AC, DominatorTree *DT) { // Limit our recursion depth. if (Depth == 6) @@ -325,6 +394,11 @@ static LinearExpression GetLinearExpression( if (!Val.canDistributeOver(NUW, NSW)) return Val; + // While we can distribute over trunc, we cannot preserve nowrap flags + // in that case. + if (Val.TruncBits) + NUW = NSW = false; + LinearExpression E(Val); switch (BOp->getOpcode()) { default: @@ -353,14 +427,11 @@ static LinearExpression GetLinearExpression( E.IsNSW &= NSW; break; } - case Instruction::Mul: { + case Instruction::Mul: E = GetLinearExpression(Val.withValue(BOp->getOperand(0)), DL, - Depth + 1, AC, DT); - E.Offset *= RHS; - E.Scale *= RHS; - E.IsNSW &= NSW; + Depth + 1, AC, DT) + .mul(RHS, NSW); break; - } case Instruction::Shl: // We're trying to linearize an expression of the kind: // shl i8 -128, 36 @@ -394,25 +465,75 @@ static LinearExpression GetLinearExpression( return Val; } -/// To ensure a pointer offset fits in an integer of size PointerSize -/// (in bits) when that size is smaller than the maximum pointer size. This is +/// To ensure a pointer offset fits in an integer of size IndexSize +/// (in bits) when that size is smaller than the maximum index size. This is /// an issue, for example, in particular for 32b pointers with negative indices /// that rely on two's complement wrap-arounds for precise alias information -/// where the maximum pointer size is 64b. -static APInt adjustToPointerSize(const APInt &Offset, unsigned PointerSize) { - assert(PointerSize <= Offset.getBitWidth() && "Invalid PointerSize!"); - unsigned ShiftBits = Offset.getBitWidth() - PointerSize; +/// where the maximum index size is 64b. +static APInt adjustToIndexSize(const APInt &Offset, unsigned IndexSize) { + assert(IndexSize <= Offset.getBitWidth() && "Invalid IndexSize!"); + unsigned ShiftBits = Offset.getBitWidth() - IndexSize; return (Offset << ShiftBits).ashr(ShiftBits); } -static unsigned getMaxPointerSize(const DataLayout &DL) { - unsigned MaxPointerSize = DL.getMaxPointerSizeInBits(); - if (MaxPointerSize < 64 && ForceAtLeast64Bits) MaxPointerSize = 64; - if (DoubleCalcBits) MaxPointerSize *= 2; +namespace { +// A linear transformation of a Value; this class represents +// ZExt(SExt(Trunc(V, TruncBits), SExtBits), ZExtBits) * Scale. +struct VariableGEPIndex { + CastedValue Val; + APInt Scale; + + // Context instruction to use when querying information about this index. + const Instruction *CxtI; - return MaxPointerSize; + /// True if all operations in this expression are NSW. + bool IsNSW; + + void dump() const { + print(dbgs()); + dbgs() << "\n"; + } + void print(raw_ostream &OS) const { + OS << "(V=" << Val.V->getName() + << ", zextbits=" << Val.ZExtBits + << ", sextbits=" << Val.SExtBits + << ", truncbits=" << Val.TruncBits + << ", scale=" << Scale << ")"; + } +}; } +// Represents the internal structure of a GEP, decomposed into a base pointer, +// constant offsets, and variable scaled indices. +struct BasicAAResult::DecomposedGEP { + // Base pointer of the GEP + const Value *Base; + // Total constant offset from base. + APInt Offset; + // Scaled variable (non-constant) indices. + SmallVector<VariableGEPIndex, 4> VarIndices; + // Are all operations inbounds GEPs or non-indexing operations? + // (None iff expression doesn't involve any geps) + Optional<bool> InBounds; + + void dump() const { + print(dbgs()); + dbgs() << "\n"; + } + void print(raw_ostream &OS) const { + OS << "(DecomposedGEP Base=" << Base->getName() + << ", Offset=" << Offset + << ", VarIndices=["; + for (size_t i = 0; i < VarIndices.size(); i++) { + if (i != 0) + OS << ", "; + VarIndices[i].print(OS); + } + OS << "])"; + } +}; + + /// If V is a symbolic pointer expression, decompose it into a base pointer /// with a constant offset and a number of scaled symbolic offsets. /// @@ -420,11 +541,6 @@ static unsigned getMaxPointerSize(const DataLayout &DL) { /// in the VarIndices vector) are Value*'s that are known to be scaled by the /// specified amount, but which may have other unrepresented high bits. As /// such, the gep cannot necessarily be reconstructed from its decomposed form. -/// -/// This function is capable of analyzing everything that getUnderlyingObject -/// can look through. To be able to do that getUnderlyingObject and -/// DecomposeGEPExpression must use the same search depth -/// (MaxLookupSearchDepth). BasicAAResult::DecomposedGEP BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL, AssumptionCache *AC, DominatorTree *DT) { @@ -433,10 +549,9 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL, SearchTimes++; const Instruction *CxtI = dyn_cast<Instruction>(V); - unsigned MaxPointerSize = getMaxPointerSize(DL); + unsigned MaxIndexSize = DL.getMaxIndexSizeInBits(); DecomposedGEP Decomposed; - Decomposed.Offset = APInt(MaxPointerSize, 0); - Decomposed.HasCompileTimeConstantScale = true; + Decomposed.Offset = APInt(MaxIndexSize, 0); do { // See if this is a bitcast or GEP. const Operator *Op = dyn_cast<Operator>(V); @@ -493,24 +608,19 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL, else if (!GEPOp->isInBounds()) Decomposed.InBounds = false; - // Don't attempt to analyze GEPs over unsized objects. - if (!GEPOp->getSourceElementType()->isSized()) { - Decomposed.Base = V; - return Decomposed; - } + assert(GEPOp->getSourceElementType()->isSized() && "GEP must be sized"); // Don't attempt to analyze GEPs if index scale is not a compile-time // constant. if (isa<ScalableVectorType>(GEPOp->getSourceElementType())) { Decomposed.Base = V; - Decomposed.HasCompileTimeConstantScale = false; return Decomposed; } unsigned AS = GEPOp->getPointerAddressSpace(); // Walk the indices of the GEP, accumulating them into BaseOff/VarIndices. gep_type_iterator GTI = gep_type_begin(GEPOp); - unsigned PointerSize = DL.getPointerSizeInBits(AS); + unsigned IndexSize = DL.getIndexSizeInBits(AS); // Assume all GEP operands are constants until proven otherwise. bool GepHasConstantOffset = true; for (User::const_op_iterator I = GEPOp->op_begin() + 1, E = GEPOp->op_end(); @@ -533,49 +643,34 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL, continue; Decomposed.Offset += DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize() * - CIdx->getValue().sextOrTrunc(MaxPointerSize); + CIdx->getValue().sextOrTrunc(MaxIndexSize); continue; } GepHasConstantOffset = false; - APInt Scale(MaxPointerSize, - DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize()); - // If the integer type is smaller than the pointer size, it is implicitly - // sign extended to pointer size. + // If the integer type is smaller than the index size, it is implicitly + // sign extended or truncated to index size. unsigned Width = Index->getType()->getIntegerBitWidth(); - unsigned SExtBits = PointerSize > Width ? PointerSize - Width : 0; + unsigned SExtBits = IndexSize > Width ? IndexSize - Width : 0; + unsigned TruncBits = IndexSize < Width ? Width - IndexSize : 0; LinearExpression LE = GetLinearExpression( - ExtendedValue(Index, 0, SExtBits), DL, 0, AC, DT); - - // The GEP index scale ("Scale") scales C1*V+C2, yielding (C1*V+C2)*Scale. - // This gives us an aggregate computation of (C1*Scale)*V + C2*Scale. + CastedValue(Index, 0, SExtBits, TruncBits), DL, 0, AC, DT); - // It can be the case that, even through C1*V+C2 does not overflow for - // relevant values of V, (C2*Scale) can overflow. In that case, we cannot - // decompose the expression in this way. - // - // FIXME: C1*Scale and the other operations in the decomposed - // (C1*Scale)*V+C2*Scale can also overflow. We should check for this - // possibility. - bool Overflow; - APInt ScaledOffset = LE.Offset.sextOrTrunc(MaxPointerSize) - .smul_ov(Scale, Overflow); - if (Overflow) { - LE = LinearExpression(ExtendedValue(Index, 0, SExtBits)); - } else { - Decomposed.Offset += ScaledOffset; - Scale *= LE.Scale.sextOrTrunc(MaxPointerSize); - } + // Scale by the type size. + unsigned TypeSize = + DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize(); + LE = LE.mul(APInt(IndexSize, TypeSize), GEPOp->isInBounds()); + Decomposed.Offset += LE.Offset.sextOrSelf(MaxIndexSize); + APInt Scale = LE.Scale.sextOrSelf(MaxIndexSize); // If we already had an occurrence of this index variable, merge this // scale into it. For example, we want to handle: // A[x][x] -> x*16 + x*4 -> x*20 // This also ensures that 'x' only appears in the index list once. for (unsigned i = 0, e = Decomposed.VarIndices.size(); i != e; ++i) { - if (Decomposed.VarIndices[i].V == LE.Val.V && - Decomposed.VarIndices[i].ZExtBits == LE.Val.ZExtBits && - Decomposed.VarIndices[i].SExtBits == LE.Val.SExtBits) { + if (Decomposed.VarIndices[i].Val.V == LE.Val.V && + Decomposed.VarIndices[i].Val.hasSameCastsAs(LE.Val)) { Scale += Decomposed.VarIndices[i].Scale; Decomposed.VarIndices.erase(Decomposed.VarIndices.begin() + i); break; @@ -583,19 +678,18 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL, } // Make sure that we have a scale that makes sense for this target's - // pointer size. - Scale = adjustToPointerSize(Scale, PointerSize); + // index size. + Scale = adjustToIndexSize(Scale, IndexSize); if (!!Scale) { - VariableGEPIndex Entry = { - LE.Val.V, LE.Val.ZExtBits, LE.Val.SExtBits, Scale, CxtI, LE.IsNSW}; + VariableGEPIndex Entry = {LE.Val, Scale, CxtI, LE.IsNSW}; Decomposed.VarIndices.push_back(Entry); } } // Take care of wrap-arounds if (GepHasConstantOffset) - Decomposed.Offset = adjustToPointerSize(Decomposed.Offset, PointerSize); + Decomposed.Offset = adjustToIndexSize(Decomposed.Offset, IndexSize); // Analyze the base pointer next. V = GEPOp->getOperand(0); @@ -838,7 +932,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, // then the call can not mod/ref the pointer unless the call takes the pointer // as an argument, and itself doesn't capture it. if (!isa<Constant>(Object) && Call != Object && - isNonEscapingLocalObject(Object, &AAQI.IsCapturedCache)) { + AAQI.CI->isNotCapturedBeforeOrAt(Object, Call)) { // Optimistically assume that call doesn't touch Object and check this // assumption in the following loop. @@ -852,8 +946,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, // pointer were passed to arguments that were neither of these, then it // couldn't be no-capture. if (!(*CI)->getType()->isPointerTy() || - (!Call->doesNotCapture(OperandNo) && - OperandNo < Call->getNumArgOperands() && + (!Call->doesNotCapture(OperandNo) && OperandNo < Call->arg_size() && !Call->isByValArgument(OperandNo))) continue; @@ -1046,20 +1139,13 @@ AliasResult BasicAAResult::aliasGEP( DecomposedGEP DecompGEP1 = DecomposeGEPExpression(GEP1, DL, &AC, DT); DecomposedGEP DecompGEP2 = DecomposeGEPExpression(V2, DL, &AC, DT); - // Don't attempt to analyze the decomposed GEP if index scale is not a - // compile-time constant. - if (!DecompGEP1.HasCompileTimeConstantScale || - !DecompGEP2.HasCompileTimeConstantScale) + // Bail if we were not able to decompose anything. + if (DecompGEP1.Base == GEP1 && DecompGEP2.Base == V2) return AliasResult::MayAlias; - assert(DecompGEP1.Base == UnderlyingV1 && DecompGEP2.Base == UnderlyingV2 && - "DecomposeGEPExpression returned a result different from " - "getUnderlyingObject"); - // Subtract the GEP2 pointer from the GEP1 pointer to find out their // symbolic difference. - DecompGEP1.Offset -= DecompGEP2.Offset; - GetIndexDifference(DecompGEP1.VarIndices, DecompGEP2.VarIndices); + subtractDecomposedGEPs(DecompGEP1, DecompGEP2); // If an inbounds GEP would have to start from an out of bounds address // for the two to alias, then we can assume noalias. @@ -1079,14 +1165,14 @@ AliasResult BasicAAResult::aliasGEP( // For GEPs with identical offsets, we can preserve the size and AAInfo // when performing the alias check on the underlying objects. if (DecompGEP1.Offset == 0 && DecompGEP1.VarIndices.empty()) - return getBestAAResults().alias( - MemoryLocation(UnderlyingV1, V1Size), - MemoryLocation(UnderlyingV2, V2Size), AAQI); + return getBestAAResults().alias(MemoryLocation(DecompGEP1.Base, V1Size), + MemoryLocation(DecompGEP2.Base, V2Size), + AAQI); // Do the base pointers alias? AliasResult BaseAlias = getBestAAResults().alias( - MemoryLocation::getBeforeOrAfter(UnderlyingV1), - MemoryLocation::getBeforeOrAfter(UnderlyingV2), AAQI); + MemoryLocation::getBeforeOrAfter(DecompGEP1.Base), + MemoryLocation::getBeforeOrAfter(DecompGEP2.Base), AAQI); // If we get a No or May, then return it immediately, no amount of analysis // will improve this situation. @@ -1100,7 +1186,7 @@ AliasResult BasicAAResult::aliasGEP( // is less than the size of the associated memory object, then we know // that the objects are partially overlapping. If the difference is // greater, we know they do not overlap. - if (DecompGEP1.Offset != 0 && DecompGEP1.VarIndices.empty()) { + if (DecompGEP1.VarIndices.empty()) { APInt &Off = DecompGEP1.Offset; // Initialize for Off >= 0 (V2 <= GEP1) case. @@ -1122,133 +1208,124 @@ AliasResult BasicAAResult::aliasGEP( Off = -Off; } - if (VLeftSize.hasValue()) { - const uint64_t LSize = VLeftSize.getValue(); - if (Off.ult(LSize)) { - // Conservatively drop processing if a phi was visited and/or offset is - // too big. - AliasResult AR = AliasResult::PartialAlias; - if (VRightSize.hasValue() && Off.ule(INT32_MAX) && - (Off + VRightSize.getValue()).ule(LSize)) { - // Memory referenced by right pointer is nested. Save the offset in - // cache. Note that originally offset estimated as GEP1-V2, but - // AliasResult contains the shift that represents GEP1+Offset=V2. - AR.setOffset(-Off.getSExtValue()); - AR.swap(Swapped); - } - return AR; + if (!VLeftSize.hasValue()) + return AliasResult::MayAlias; + + const uint64_t LSize = VLeftSize.getValue(); + if (Off.ult(LSize)) { + // Conservatively drop processing if a phi was visited and/or offset is + // too big. + AliasResult AR = AliasResult::PartialAlias; + if (VRightSize.hasValue() && Off.ule(INT32_MAX) && + (Off + VRightSize.getValue()).ule(LSize)) { + // Memory referenced by right pointer is nested. Save the offset in + // cache. Note that originally offset estimated as GEP1-V2, but + // AliasResult contains the shift that represents GEP1+Offset=V2. + AR.setOffset(-Off.getSExtValue()); + AR.swap(Swapped); } - return AliasResult::NoAlias; + return AR; } + return AliasResult::NoAlias; } - if (!DecompGEP1.VarIndices.empty()) { - APInt GCD; - bool AllNonNegative = DecompGEP1.Offset.isNonNegative(); - bool AllNonPositive = DecompGEP1.Offset.isNonPositive(); - for (unsigned i = 0, e = DecompGEP1.VarIndices.size(); i != e; ++i) { - APInt Scale = DecompGEP1.VarIndices[i].Scale; - APInt ScaleForGCD = DecompGEP1.VarIndices[i].Scale; - if (!DecompGEP1.VarIndices[i].IsNSW) - ScaleForGCD = APInt::getOneBitSet(Scale.getBitWidth(), - Scale.countTrailingZeros()); - - if (i == 0) - GCD = ScaleForGCD.abs(); - else - GCD = APIntOps::GreatestCommonDivisor(GCD, ScaleForGCD.abs()); - - if (AllNonNegative || AllNonPositive) { - // If the Value could change between cycles, then any reasoning about - // the Value this cycle may not hold in the next cycle. We'll just - // give up if we can't determine conditions that hold for every cycle: - const Value *V = DecompGEP1.VarIndices[i].V; - const Instruction *CxtI = DecompGEP1.VarIndices[i].CxtI; + // We need to know both acess sizes for all the following heuristics. + if (!V1Size.hasValue() || !V2Size.hasValue()) + return AliasResult::MayAlias; - KnownBits Known = computeKnownBits(V, DL, 0, &AC, CxtI, DT); - bool SignKnownZero = Known.isNonNegative(); - bool SignKnownOne = Known.isNegative(); + APInt GCD; + ConstantRange OffsetRange = ConstantRange(DecompGEP1.Offset); + for (unsigned i = 0, e = DecompGEP1.VarIndices.size(); i != e; ++i) { + const VariableGEPIndex &Index = DecompGEP1.VarIndices[i]; + const APInt &Scale = Index.Scale; + APInt ScaleForGCD = Scale; + if (!Index.IsNSW) + ScaleForGCD = APInt::getOneBitSet(Scale.getBitWidth(), + Scale.countTrailingZeros()); - // Zero-extension widens the variable, and so forces the sign - // bit to zero. - bool IsZExt = DecompGEP1.VarIndices[i].ZExtBits > 0 || isa<ZExtInst>(V); - SignKnownZero |= IsZExt; - SignKnownOne &= !IsZExt; + if (i == 0) + GCD = ScaleForGCD.abs(); + else + GCD = APIntOps::GreatestCommonDivisor(GCD, ScaleForGCD.abs()); - AllNonNegative &= (SignKnownZero && Scale.isNonNegative()) || - (SignKnownOne && Scale.isNonPositive()); - AllNonPositive &= (SignKnownZero && Scale.isNonPositive()) || - (SignKnownOne && Scale.isNonNegative()); - } - } + ConstantRange CR = + computeConstantRange(Index.Val.V, true, &AC, Index.CxtI); + KnownBits Known = + computeKnownBits(Index.Val.V, DL, 0, &AC, Index.CxtI, DT); + CR = CR.intersectWith( + ConstantRange::fromKnownBits(Known, /* Signed */ true), + ConstantRange::Signed); + CR = Index.Val.evaluateWith(CR).sextOrTrunc(OffsetRange.getBitWidth()); - // We now have accesses at two offsets from the same base: - // 1. (...)*GCD + DecompGEP1.Offset with size V1Size - // 2. 0 with size V2Size - // Using arithmetic modulo GCD, the accesses are at - // [ModOffset..ModOffset+V1Size) and [0..V2Size). If the first access fits - // into the range [V2Size..GCD), then we know they cannot overlap. - APInt ModOffset = DecompGEP1.Offset.srem(GCD); - if (ModOffset.isNegative()) - ModOffset += GCD; // We want mod, not rem. - if (V1Size.hasValue() && V2Size.hasValue() && - ModOffset.uge(V2Size.getValue()) && - (GCD - ModOffset).uge(V1Size.getValue())) - return AliasResult::NoAlias; + assert(OffsetRange.getBitWidth() == Scale.getBitWidth() && + "Bit widths are normalized to MaxIndexSize"); + if (Index.IsNSW) + OffsetRange = OffsetRange.add(CR.smul_sat(ConstantRange(Scale))); + else + OffsetRange = OffsetRange.add(CR.smul_fast(ConstantRange(Scale))); + } - // If we know all the variables are non-negative, then the total offset is - // also non-negative and >= DecompGEP1.Offset. We have the following layout: - // [0, V2Size) ... [TotalOffset, TotalOffer+V1Size] - // If DecompGEP1.Offset >= V2Size, the accesses don't alias. - if (AllNonNegative && V2Size.hasValue() && - DecompGEP1.Offset.uge(V2Size.getValue())) - return AliasResult::NoAlias; - // Similarly, if the variables are non-positive, then the total offset is - // also non-positive and <= DecompGEP1.Offset. We have the following layout: - // [TotalOffset, TotalOffset+V1Size) ... [0, V2Size) - // If -DecompGEP1.Offset >= V1Size, the accesses don't alias. - if (AllNonPositive && V1Size.hasValue() && - (-DecompGEP1.Offset).uge(V1Size.getValue())) - return AliasResult::NoAlias; + // We now have accesses at two offsets from the same base: + // 1. (...)*GCD + DecompGEP1.Offset with size V1Size + // 2. 0 with size V2Size + // Using arithmetic modulo GCD, the accesses are at + // [ModOffset..ModOffset+V1Size) and [0..V2Size). If the first access fits + // into the range [V2Size..GCD), then we know they cannot overlap. + APInt ModOffset = DecompGEP1.Offset.srem(GCD); + if (ModOffset.isNegative()) + ModOffset += GCD; // We want mod, not rem. + if (ModOffset.uge(V2Size.getValue()) && + (GCD - ModOffset).uge(V1Size.getValue())) + return AliasResult::NoAlias; - if (V1Size.hasValue() && V2Size.hasValue()) { - // Try to determine whether abs(VarIndex) > 0. - Optional<APInt> MinAbsVarIndex; - if (DecompGEP1.VarIndices.size() == 1) { - // VarIndex = Scale*V. If V != 0 then abs(VarIndex) >= abs(Scale). - const VariableGEPIndex &Var = DecompGEP1.VarIndices[0]; - if (isKnownNonZero(Var.V, DL, 0, &AC, Var.CxtI, DT)) - MinAbsVarIndex = Var.Scale.abs(); - } else if (DecompGEP1.VarIndices.size() == 2) { - // VarIndex = Scale*V0 + (-Scale)*V1. - // If V0 != V1 then abs(VarIndex) >= abs(Scale). - // Check that VisitedPhiBBs is empty, to avoid reasoning about - // inequality of values across loop iterations. - const VariableGEPIndex &Var0 = DecompGEP1.VarIndices[0]; - const VariableGEPIndex &Var1 = DecompGEP1.VarIndices[1]; - if (Var0.Scale == -Var1.Scale && Var0.ZExtBits == Var1.ZExtBits && - Var0.SExtBits == Var1.SExtBits && VisitedPhiBBs.empty() && - isKnownNonEqual(Var0.V, Var1.V, DL, &AC, /* CxtI */ nullptr, DT)) - MinAbsVarIndex = Var0.Scale.abs(); - } + // Compute ranges of potentially accessed bytes for both accesses. If the + // interseciton is empty, there can be no overlap. + unsigned BW = OffsetRange.getBitWidth(); + ConstantRange Range1 = OffsetRange.add( + ConstantRange(APInt(BW, 0), APInt(BW, V1Size.getValue()))); + ConstantRange Range2 = + ConstantRange(APInt(BW, 0), APInt(BW, V2Size.getValue())); + if (Range1.intersectWith(Range2).isEmptySet()) + return AliasResult::NoAlias; - if (MinAbsVarIndex) { - // The constant offset will have added at least +/-MinAbsVarIndex to it. - APInt OffsetLo = DecompGEP1.Offset - *MinAbsVarIndex; - APInt OffsetHi = DecompGEP1.Offset + *MinAbsVarIndex; - // Check that an access at OffsetLo or lower, and an access at OffsetHi - // or higher both do not alias. - if (OffsetLo.isNegative() && (-OffsetLo).uge(V1Size.getValue()) && - OffsetHi.isNonNegative() && OffsetHi.uge(V2Size.getValue())) - return AliasResult::NoAlias; - } + // Try to determine the range of values for VarIndex such that + // VarIndex <= -MinAbsVarIndex || MinAbsVarIndex <= VarIndex. + Optional<APInt> MinAbsVarIndex; + if (DecompGEP1.VarIndices.size() == 1) { + // VarIndex = Scale*V. + const VariableGEPIndex &Var = DecompGEP1.VarIndices[0]; + if (Var.Val.TruncBits == 0 && + isKnownNonZero(Var.Val.V, DL, 0, &AC, Var.CxtI, DT)) { + // If V != 0 then abs(VarIndex) >= abs(Scale). + MinAbsVarIndex = Var.Scale.abs(); } + } else if (DecompGEP1.VarIndices.size() == 2) { + // VarIndex = Scale*V0 + (-Scale)*V1. + // If V0 != V1 then abs(VarIndex) >= abs(Scale). + // Check that VisitedPhiBBs is empty, to avoid reasoning about + // inequality of values across loop iterations. + const VariableGEPIndex &Var0 = DecompGEP1.VarIndices[0]; + const VariableGEPIndex &Var1 = DecompGEP1.VarIndices[1]; + if (Var0.Scale == -Var1.Scale && Var0.Val.TruncBits == 0 && + Var0.Val.hasSameCastsAs(Var1.Val) && VisitedPhiBBs.empty() && + isKnownNonEqual(Var0.Val.V, Var1.Val.V, DL, &AC, /* CxtI */ nullptr, + DT)) + MinAbsVarIndex = Var0.Scale.abs(); + } - if (constantOffsetHeuristic(DecompGEP1.VarIndices, V1Size, V2Size, - DecompGEP1.Offset, &AC, DT)) + if (MinAbsVarIndex) { + // The constant offset will have added at least +/-MinAbsVarIndex to it. + APInt OffsetLo = DecompGEP1.Offset - *MinAbsVarIndex; + APInt OffsetHi = DecompGEP1.Offset + *MinAbsVarIndex; + // We know that Offset <= OffsetLo || Offset >= OffsetHi + if (OffsetLo.isNegative() && (-OffsetLo).uge(V1Size.getValue()) && + OffsetHi.isNonNegative() && OffsetHi.uge(V2Size.getValue())) return AliasResult::NoAlias; } + if (constantOffsetHeuristic(DecompGEP1, V1Size, V2Size, &AC, DT)) + return AliasResult::NoAlias; + // Statically, we can see that the base objects are the same, but the // pointers have dynamic offsets which we can't resolve. And none of our // little tricks above worked. @@ -1517,10 +1594,10 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, // location if that memory location doesn't escape. Or it may pass a // nocapture value to other functions as long as they don't capture it. if (isEscapeSource(O1) && - isNonEscapingLocalObject(O2, &AAQI.IsCapturedCache)) + AAQI.CI->isNotCapturedBeforeOrAt(O2, cast<Instruction>(O1))) return AliasResult::NoAlias; if (isEscapeSource(O2) && - isNonEscapingLocalObject(O1, &AAQI.IsCapturedCache)) + AAQI.CI->isNotCapturedBeforeOrAt(O1, cast<Instruction>(O2))) return AliasResult::NoAlias; } @@ -1692,62 +1769,54 @@ bool BasicAAResult::isValueEqualInPotentialCycles(const Value *V, } /// Computes the symbolic difference between two de-composed GEPs. -/// -/// Dest and Src are the variable indices from two decomposed GetElementPtr -/// instructions GEP1 and GEP2 which have common base pointers. -void BasicAAResult::GetIndexDifference( - SmallVectorImpl<VariableGEPIndex> &Dest, - const SmallVectorImpl<VariableGEPIndex> &Src) { - if (Src.empty()) - return; - - for (unsigned i = 0, e = Src.size(); i != e; ++i) { - const Value *V = Src[i].V; - unsigned ZExtBits = Src[i].ZExtBits, SExtBits = Src[i].SExtBits; - APInt Scale = Src[i].Scale; - +void BasicAAResult::subtractDecomposedGEPs(DecomposedGEP &DestGEP, + const DecomposedGEP &SrcGEP) { + DestGEP.Offset -= SrcGEP.Offset; + for (const VariableGEPIndex &Src : SrcGEP.VarIndices) { // Find V in Dest. This is N^2, but pointer indices almost never have more // than a few variable indexes. - for (unsigned j = 0, e = Dest.size(); j != e; ++j) { - if (!isValueEqualInPotentialCycles(Dest[j].V, V) || - Dest[j].ZExtBits != ZExtBits || Dest[j].SExtBits != SExtBits) + bool Found = false; + for (auto I : enumerate(DestGEP.VarIndices)) { + VariableGEPIndex &Dest = I.value(); + if (!isValueEqualInPotentialCycles(Dest.Val.V, Src.Val.V) || + !Dest.Val.hasSameCastsAs(Src.Val)) continue; // If we found it, subtract off Scale V's from the entry in Dest. If it // goes to zero, remove the entry. - if (Dest[j].Scale != Scale) { - Dest[j].Scale -= Scale; - Dest[j].IsNSW = false; - } else - Dest.erase(Dest.begin() + j); - Scale = 0; + if (Dest.Scale != Src.Scale) { + Dest.Scale -= Src.Scale; + Dest.IsNSW = false; + } else { + DestGEP.VarIndices.erase(DestGEP.VarIndices.begin() + I.index()); + } + Found = true; break; } // If we didn't consume this entry, add it to the end of the Dest list. - if (!!Scale) { - VariableGEPIndex Entry = {V, ZExtBits, SExtBits, - -Scale, Src[i].CxtI, Src[i].IsNSW}; - Dest.push_back(Entry); + if (!Found) { + VariableGEPIndex Entry = {Src.Val, -Src.Scale, Src.CxtI, Src.IsNSW}; + DestGEP.VarIndices.push_back(Entry); } } } bool BasicAAResult::constantOffsetHeuristic( - const SmallVectorImpl<VariableGEPIndex> &VarIndices, - LocationSize MaybeV1Size, LocationSize MaybeV2Size, const APInt &BaseOffset, - AssumptionCache *AC, DominatorTree *DT) { - if (VarIndices.size() != 2 || !MaybeV1Size.hasValue() || + const DecomposedGEP &GEP, LocationSize MaybeV1Size, + LocationSize MaybeV2Size, AssumptionCache *AC, DominatorTree *DT) { + if (GEP.VarIndices.size() != 2 || !MaybeV1Size.hasValue() || !MaybeV2Size.hasValue()) return false; const uint64_t V1Size = MaybeV1Size.getValue(); const uint64_t V2Size = MaybeV2Size.getValue(); - const VariableGEPIndex &Var0 = VarIndices[0], &Var1 = VarIndices[1]; + const VariableGEPIndex &Var0 = GEP.VarIndices[0], &Var1 = GEP.VarIndices[1]; - if (Var0.ZExtBits != Var1.ZExtBits || Var0.SExtBits != Var1.SExtBits || - Var0.Scale != -Var1.Scale || Var0.V->getType() != Var1.V->getType()) + if (Var0.Val.TruncBits != 0 || !Var0.Val.hasSameCastsAs(Var1.Val) || + Var0.Scale != -Var1.Scale || + Var0.Val.V->getType() != Var1.Val.V->getType()) return false; // We'll strip off the Extensions of Var0 and Var1 and do another round @@ -1755,11 +1824,10 @@ bool BasicAAResult::constantOffsetHeuristic( // is zext(%x + 1) we should get V1 == %x and V1Offset == 1. LinearExpression E0 = - GetLinearExpression(ExtendedValue(Var0.V), DL, 0, AC, DT); + GetLinearExpression(CastedValue(Var0.Val.V), DL, 0, AC, DT); LinearExpression E1 = - GetLinearExpression(ExtendedValue(Var1.V), DL, 0, AC, DT); - if (E0.Scale != E1.Scale || E0.Val.ZExtBits != E1.Val.ZExtBits || - E0.Val.SExtBits != E1.Val.SExtBits || + GetLinearExpression(CastedValue(Var1.Val.V), DL, 0, AC, DT); + if (E0.Scale != E1.Scale || !E0.Val.hasSameCastsAs(E1.Val) || !isValueEqualInPotentialCycles(E0.Val.V, E1.Val.V)) return false; @@ -1779,8 +1847,8 @@ bool BasicAAResult::constantOffsetHeuristic( // arithmetic (i.e. for some values of GEP1 and V2 GEP1 < V2, and for other // values GEP1 > V2). We'll therefore only declare NoAlias if both V1Size and // V2Size can fit in the MinDiffBytes gap. - return MinDiffBytes.uge(V1Size + BaseOffset.abs()) && - MinDiffBytes.uge(V2Size + BaseOffset.abs()); + return MinDiffBytes.uge(V1Size + GEP.Offset.abs()) && + MinDiffBytes.uge(V2Size + GEP.Offset.abs()); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp index e4e45b3076be..2a5e1f65d731 100644 --- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp @@ -602,7 +602,7 @@ BlockFrequencyInfoImplBase::getProfileCountFromFreq(const Function &F, if (!EntryCount) return None; // Use 128 bit APInt to do the arithmetic to avoid overflow. - APInt BlockCount(128, EntryCount.getCount()); + APInt BlockCount(128, EntryCount->getCount()); APInt BlockFreq(128, Freq); APInt EntryFreq(128, getEntryFreq()); BlockCount *= BlockFreq; diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp index aa6b93fe3f07..33fdc8b628c5 100644 --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -190,7 +190,7 @@ void BranchProbabilityInfo::SccInfo::getSccExitBlocks( if (isSCCExitingBlock(BB, SccNum)) for (const auto *Succ : successors(BB)) if (getSCCNum(Succ) != SccNum) - Exits.push_back(const_cast<BasicBlock *>(BB)); + Exits.push_back(const_cast<BasicBlock *>(Succ)); } } diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp index 253cc0b0a579..c60b70ae5b69 100644 --- a/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -38,12 +38,13 @@ using namespace llvm; // Explicit template instantiations and specialization definitions for core // template typedefs. namespace llvm { - static cl::opt<bool> AbortOnMaxDevirtIterationsReached( "abort-on-max-devirt-iterations-reached", cl::desc("Abort when the max iterations for devirtualization CGSCC repeat " "pass is reached")); +AnalysisKey ShouldNotRunFunctionPassesAnalysis::Key; + // Explicit instantiations for the core proxy templates. template class AllAnalysesOn<LazyCallGraph::SCC>; template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>; @@ -119,12 +120,6 @@ PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, // Finally, we intersect the final preserved analyses to compute the // aggregate preserved set for this pass manager. PA.intersect(std::move(PassPA)); - - // FIXME: Historically, the pass managers all called the LLVM context's - // yield function here. We don't have a generic way to acquire the - // context and it isn't yet clear what the right pattern is for yielding - // in the new pass manager so it is currently omitted. - // ...getContext().yield(); } // Before we mark all of *this* SCC's analyses as preserved below, intersect @@ -547,6 +542,9 @@ PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C, Function &F = N->getFunction(); + if (NoRerun && FAM.getCachedResult<ShouldNotRunFunctionPassesAnalysis>(F)) + continue; + PassInstrumentation PI = FAM.getResult<PassInstrumentationAnalysis>(F); if (!PI.runBeforePass<Function>(*Pass, F)) continue; @@ -562,7 +560,9 @@ PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C, // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, PassPA); + FAM.invalidate(F, EagerlyInvalidate ? PreservedAnalyses::none() : PassPA); + if (NoRerun) + (void)FAM.getResult<ShouldNotRunFunctionPassesAnalysis>(F); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -863,7 +863,7 @@ incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G, // split-off SCCs. // We know however that this will preserve any FAM proxy so go ahead and mark // that. - PreservedAnalyses PA; + auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>(); PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); AM.invalidate(*OldC, PA); diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp index 5fe4f9befc86..8955658cb9e7 100644 --- a/llvm/lib/Analysis/CaptureTracking.cpp +++ b/llvm/lib/Analysis/CaptureTracking.cpp @@ -98,10 +98,10 @@ namespace { /// as the given instruction and the use. struct CapturesBefore : public CaptureTracker { - CapturesBefore(bool ReturnCaptures, const Instruction *I, const DominatorTree *DT, - bool IncludeI) - : BeforeHere(I), DT(DT), - ReturnCaptures(ReturnCaptures), IncludeI(IncludeI), Captured(false) {} + CapturesBefore(bool ReturnCaptures, const Instruction *I, + const DominatorTree *DT, bool IncludeI, const LoopInfo *LI) + : BeforeHere(I), DT(DT), ReturnCaptures(ReturnCaptures), + IncludeI(IncludeI), Captured(false), LI(LI) {} void tooManyUses() override { Captured = true; } @@ -115,7 +115,7 @@ namespace { return true; // Check whether there is a path from I to BeforeHere. - return !isPotentiallyReachable(I, BeforeHere, nullptr, DT); + return !isPotentiallyReachable(I, BeforeHere, nullptr, DT, LI); } bool captured(const Use *U) override { @@ -140,6 +140,68 @@ namespace { bool IncludeI; bool Captured; + + const LoopInfo *LI; + }; + + /// Find the 'earliest' instruction before which the pointer is known not to + /// be captured. Here an instruction A is considered earlier than instruction + /// B, if A dominates B. If 2 escapes do not dominate each other, the + /// terminator of the common dominator is chosen. If not all uses cannot be + /// analyzed, the earliest escape is set to the first instruction in the + /// function entry block. + // NOTE: Users have to make sure instructions compared against the earliest + // escape are not in a cycle. + struct EarliestCaptures : public CaptureTracker { + + EarliestCaptures(bool ReturnCaptures, Function &F, const DominatorTree &DT) + : DT(DT), ReturnCaptures(ReturnCaptures), Captured(false), F(F) {} + + void tooManyUses() override { + Captured = true; + EarliestCapture = &*F.getEntryBlock().begin(); + } + + bool captured(const Use *U) override { + Instruction *I = cast<Instruction>(U->getUser()); + if (isa<ReturnInst>(I) && !ReturnCaptures) + return false; + + if (!EarliestCapture) { + EarliestCapture = I; + } else if (EarliestCapture->getParent() == I->getParent()) { + if (I->comesBefore(EarliestCapture)) + EarliestCapture = I; + } else { + BasicBlock *CurrentBB = I->getParent(); + BasicBlock *EarliestBB = EarliestCapture->getParent(); + if (DT.dominates(EarliestBB, CurrentBB)) { + // EarliestCapture already comes before the current use. + } else if (DT.dominates(CurrentBB, EarliestBB)) { + EarliestCapture = I; + } else { + // Otherwise find the nearest common dominator and use its terminator. + auto *NearestCommonDom = + DT.findNearestCommonDominator(CurrentBB, EarliestBB); + EarliestCapture = NearestCommonDom->getTerminator(); + } + } + Captured = true; + + // Return false to continue analysis; we need to see all potential + // captures. + return false; + } + + Instruction *EarliestCapture = nullptr; + + const DominatorTree &DT; + + bool ReturnCaptures; + + bool Captured; + + Function &F; }; } @@ -183,7 +245,8 @@ bool llvm::PointerMayBeCaptured(const Value *V, bool llvm::PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, bool StoreCaptures, const Instruction *I, const DominatorTree *DT, bool IncludeI, - unsigned MaxUsesToExplore) { + unsigned MaxUsesToExplore, + const LoopInfo *LI) { assert(!isa<GlobalValue>(V) && "It doesn't make sense to ask whether a global is captured."); @@ -194,7 +257,7 @@ bool llvm::PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, // TODO: See comment in PointerMayBeCaptured regarding what could be done // with StoreCaptures. - CapturesBefore CB(ReturnCaptures, I, DT, IncludeI); + CapturesBefore CB(ReturnCaptures, I, DT, IncludeI, LI); PointerMayBeCaptured(V, &CB, MaxUsesToExplore); if (CB.Captured) ++NumCapturedBefore; @@ -203,6 +266,22 @@ bool llvm::PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, return CB.Captured; } +Instruction *llvm::FindEarliestCapture(const Value *V, Function &F, + bool ReturnCaptures, bool StoreCaptures, + const DominatorTree &DT, + unsigned MaxUsesToExplore) { + assert(!isa<GlobalValue>(V) && + "It doesn't make sense to ask whether a global is captured."); + + EarliestCaptures CB(ReturnCaptures, F, DT); + PointerMayBeCaptured(V, &CB, MaxUsesToExplore); + if (CB.Captured) + ++NumCapturedBefore; + else + ++NumNotCapturedBefore; + return CB.EarliestCapture; +} + void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker, unsigned MaxUsesToExplore) { assert(V->getType()->isPointerTy() && "Capture is for pointers only!"); diff --git a/llvm/lib/Analysis/CmpInstAnalysis.cpp b/llvm/lib/Analysis/CmpInstAnalysis.cpp index a5757be2c4f4..5b951980a0aa 100644 --- a/llvm/lib/Analysis/CmpInstAnalysis.cpp +++ b/llvm/lib/Analysis/CmpInstAnalysis.cpp @@ -77,28 +77,28 @@ bool llvm::decomposeBitTestICmp(Value *LHS, Value *RHS, return false; case ICmpInst::ICMP_SLT: // X < 0 is equivalent to (X & SignMask) != 0. - if (!C->isNullValue()) + if (!C->isZero()) return false; Mask = APInt::getSignMask(C->getBitWidth()); Pred = ICmpInst::ICMP_NE; break; case ICmpInst::ICMP_SLE: // X <= -1 is equivalent to (X & SignMask) != 0. - if (!C->isAllOnesValue()) + if (!C->isAllOnes()) return false; Mask = APInt::getSignMask(C->getBitWidth()); Pred = ICmpInst::ICMP_NE; break; case ICmpInst::ICMP_SGT: // X > -1 is equivalent to (X & SignMask) == 0. - if (!C->isAllOnesValue()) + if (!C->isAllOnes()) return false; Mask = APInt::getSignMask(C->getBitWidth()); Pred = ICmpInst::ICMP_EQ; break; case ICmpInst::ICMP_SGE: // X >= 0 is equivalent to (X & SignMask) == 0. - if (!C->isNullValue()) + if (!C->isZero()) return false; Mask = APInt::getSignMask(C->getBitWidth()); Pred = ICmpInst::ICMP_EQ; diff --git a/llvm/lib/Analysis/CodeMetrics.cpp b/llvm/lib/Analysis/CodeMetrics.cpp index 8c8e2ee6627f..27c52506352f 100644 --- a/llvm/lib/Analysis/CodeMetrics.cpp +++ b/llvm/lib/Analysis/CodeMetrics.cpp @@ -34,8 +34,9 @@ appendSpeculatableOperands(const Value *V, for (const Value *Operand : U->operands()) if (Visited.insert(Operand).second) - if (isSafeToSpeculativelyExecute(Operand)) - Worklist.push_back(Operand); + if (const auto *I = dyn_cast<Instruction>(Operand)) + if (!I->mayHaveSideEffects() && !I->isTerminator()) + Worklist.push_back(I); } static void completeEphemeralValues(SmallPtrSetImpl<const Value *> &Visited, diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp index b28a0d6c78cd..3ed3b8902343 100644 --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -63,11 +63,6 @@ using namespace llvm; namespace { -Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, - ArrayRef<Constant *> Ops, - const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool ForLoadOperand); //===----------------------------------------------------------------------===// // Constant Folding internal helper functions @@ -357,9 +352,9 @@ Constant *llvm::ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy, const DataLayout &DL) { do { Type *SrcTy = C->getType(); - uint64_t DestSize = DL.getTypeSizeInBits(DestTy); - uint64_t SrcSize = DL.getTypeSizeInBits(SrcTy); - if (SrcSize < DestSize) + TypeSize DestSize = DL.getTypeSizeInBits(DestTy); + TypeSize SrcSize = DL.getTypeSizeInBits(SrcTy); + if (!TypeSize::isKnownGE(SrcSize, DestSize)) return nullptr; // Catch the obvious splat cases (since all-zeros can coerce non-integral @@ -550,19 +545,16 @@ bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr, return false; } -Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, - const DataLayout &DL) { +Constant *FoldReinterpretLoadFromConst(Constant *C, Type *LoadTy, + int64_t Offset, const DataLayout &DL) { // Bail out early. Not expect to load from scalable global variable. if (isa<ScalableVectorType>(LoadTy)) return nullptr; - auto *PTy = cast<PointerType>(C->getType()); auto *IntType = dyn_cast<IntegerType>(LoadTy); // If this isn't an integer load we can't fold it directly. if (!IntType) { - unsigned AS = PTy->getAddressSpace(); - // If this is a float/double load, we can try folding it as an int32/64 load // and then bitcast the result. This can be useful for union cases. Note // that address spaces don't matter here since we're not going to result in @@ -580,8 +572,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, } else return nullptr; - C = FoldBitCast(C, MapTy->getPointerTo(AS), DL); - if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, MapTy, DL)) { + if (Constant *Res = FoldReinterpretLoadFromConst(C, MapTy, Offset, DL)) { if (Res->isNullValue() && !LoadTy->isX86_MMXTy() && !LoadTy->isX86_AMXTy()) // Materializing a zero can be done trivially without a bitcast @@ -607,19 +598,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, if (BytesLoaded > 32 || BytesLoaded == 0) return nullptr; - GlobalValue *GVal; - APInt OffsetAI; - if (!IsConstantOffsetFromGlobal(C, GVal, OffsetAI, DL)) - return nullptr; - - auto *GV = dyn_cast<GlobalVariable>(GVal); - if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || - !GV->getInitializer()->getType()->isSized()) - return nullptr; - - int64_t Offset = OffsetAI.getSExtValue(); - int64_t InitializerSize = - DL.getTypeAllocSize(GV->getInitializer()->getType()).getFixedSize(); + int64_t InitializerSize = DL.getTypeAllocSize(C->getType()).getFixedSize(); // If we're not accessing anything in this constant, the result is undefined. if (Offset <= -1 * static_cast<int64_t>(BytesLoaded)) @@ -640,7 +619,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, Offset = 0; } - if (!ReadDataFromGlobal(GV->getInitializer(), Offset, CurPtr, BytesLeft, DL)) + if (!ReadDataFromGlobal(C, Offset, CurPtr, BytesLeft, DL)) return nullptr; APInt ResultVal = APInt(IntType->getBitWidth(), 0); @@ -661,111 +640,70 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, return ConstantInt::get(IntType->getContext(), ResultVal); } -Constant *ConstantFoldLoadThroughBitcastExpr(ConstantExpr *CE, Type *DestTy, - const DataLayout &DL) { - auto *SrcPtr = CE->getOperand(0); - if (!SrcPtr->getType()->isPointerTy()) +/// If this Offset points exactly to the start of an aggregate element, return +/// that element, otherwise return nullptr. +Constant *getConstantAtOffset(Constant *Base, APInt Offset, + const DataLayout &DL) { + if (Offset.isZero()) + return Base; + + if (!isa<ConstantAggregate>(Base) && !isa<ConstantDataSequential>(Base)) + return nullptr; + + Type *ElemTy = Base->getType(); + SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset); + if (!Offset.isZero() || !Indices[0].isZero()) return nullptr; - return ConstantFoldLoadFromConstPtr(SrcPtr, DestTy, DL); + Constant *C = Base; + for (const APInt &Index : drop_begin(Indices)) { + if (Index.isNegative() || Index.getActiveBits() >= 32) + return nullptr; + + C = C->getAggregateElement(Index.getZExtValue()); + if (!C) + return nullptr; + } + + return C; } } // end anonymous namespace -Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, - const DataLayout &DL) { - // First, try the easy cases: - if (auto *GV = dyn_cast<GlobalVariable>(C)) - if (GV->isConstant() && GV->hasDefinitiveInitializer()) - return ConstantFoldLoadThroughBitcast(GV->getInitializer(), Ty, DL); - - if (auto *GA = dyn_cast<GlobalAlias>(C)) - if (GA->getAliasee() && !GA->isInterposable()) - return ConstantFoldLoadFromConstPtr(GA->getAliasee(), Ty, DL); +Constant *llvm::ConstantFoldLoadFromConst(Constant *C, Type *Ty, + const APInt &Offset, + const DataLayout &DL) { + if (Constant *AtOffset = getConstantAtOffset(C, Offset, DL)) + if (Constant *Result = ConstantFoldLoadThroughBitcast(AtOffset, Ty, DL)) + return Result; - // If the loaded value isn't a constant expr, we can't handle it. - auto *CE = dyn_cast<ConstantExpr>(C); - if (!CE) - return nullptr; + // Try hard to fold loads from bitcasted strange and non-type-safe things. + if (Offset.getMinSignedBits() <= 64) + return FoldReinterpretLoadFromConst(C, Ty, Offset.getSExtValue(), DL); - if (CE->getOpcode() == Instruction::GetElementPtr) { - if (auto *GV = dyn_cast<GlobalVariable>(CE->getOperand(0))) { - if (GV->isConstant() && GV->hasDefinitiveInitializer()) { - if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr( - GV->getInitializer(), CE, Ty, DL)) - return V; - } - } else { - // Try to simplify GEP if the pointer operand wasn't a GlobalVariable. - // SymbolicallyEvaluateGEP() with `ForLoadOperand = true` can potentially - // simplify the GEP more than it normally would have been, but should only - // be used for const folding loads. - SmallVector<Constant *> Ops; - for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I) - Ops.push_back(cast<Constant>(CE->getOperand(I))); - if (auto *Simplified = dyn_cast_or_null<ConstantExpr>( - SymbolicallyEvaluateGEP(cast<GEPOperator>(CE), Ops, DL, nullptr, - /*ForLoadOperand*/ true))) { - // If the symbolically evaluated GEP is another GEP, we can only const - // fold it if the resulting pointer operand is a GlobalValue. Otherwise - // there is nothing else to simplify since the GEP is already in the - // most simplified form. - if (isa<GEPOperator>(Simplified)) { - if (auto *GV = dyn_cast<GlobalVariable>(Simplified->getOperand(0))) { - if (GV->isConstant() && GV->hasDefinitiveInitializer()) { - if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr( - GV->getInitializer(), Simplified, Ty, DL)) - return V; - } - } - } else { - return ConstantFoldLoadFromConstPtr(Simplified, Ty, DL); - } - } - } - } + return nullptr; +} - if (CE->getOpcode() == Instruction::BitCast) - if (Constant *LoadedC = ConstantFoldLoadThroughBitcastExpr(CE, Ty, DL)) - return LoadedC; +Constant *llvm::ConstantFoldLoadFromConst(Constant *C, Type *Ty, + const DataLayout &DL) { + return ConstantFoldLoadFromConst(C, Ty, APInt(64, 0), DL); +} - // Instead of loading constant c string, use corresponding integer value - // directly if string length is small enough. - StringRef Str; - if (getConstantStringInfo(CE, Str) && !Str.empty()) { - size_t StrLen = Str.size(); - unsigned NumBits = Ty->getPrimitiveSizeInBits(); - // Replace load with immediate integer if the result is an integer or fp - // value. - if ((NumBits >> 3) == StrLen + 1 && (NumBits & 7) == 0 && - (isa<IntegerType>(Ty) || Ty->isFloatingPointTy())) { - APInt StrVal(NumBits, 0); - APInt SingleChar(NumBits, 0); - if (DL.isLittleEndian()) { - for (unsigned char C : reverse(Str.bytes())) { - SingleChar = static_cast<uint64_t>(C); - StrVal = (StrVal << 8) | SingleChar; - } - } else { - for (unsigned char C : Str.bytes()) { - SingleChar = static_cast<uint64_t>(C); - StrVal = (StrVal << 8) | SingleChar; - } - // Append NULL at the end. - SingleChar = 0; - StrVal = (StrVal << 8) | SingleChar; - } +Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, + APInt Offset, + const DataLayout &DL) { + C = cast<Constant>(C->stripAndAccumulateConstantOffsets( + DL, Offset, /* AllowNonInbounds */ true)); - Constant *Res = ConstantInt::get(CE->getContext(), StrVal); - if (Ty->isFloatingPointTy()) - Res = ConstantExpr::getBitCast(Res, Ty); - return Res; - } - } + if (auto *GV = dyn_cast<GlobalVariable>(C)) + if (GV->isConstant() && GV->hasDefinitiveInitializer()) + if (Constant *Result = ConstantFoldLoadFromConst(GV->getInitializer(), Ty, + Offset, DL)) + return Result; // If this load comes from anywhere in a constant global, and if the global // is all undef or zero, we know what it loads. - if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(CE))) { + if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(C))) { if (GV->isConstant() && GV->hasDefinitiveInitializer()) { if (GV->getInitializer()->isNullValue()) return Constant::getNullValue(Ty); @@ -774,8 +712,13 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, } } - // Try hard to fold loads from bitcasted strange and non-type-safe things. - return FoldReinterpretLoadFromConstPtr(CE, Ty, DL); + return nullptr; +} + +Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, + const DataLayout &DL) { + APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0); + return ConstantFoldLoadFromConstPtr(C, Ty, Offset, DL); } namespace { @@ -795,11 +738,11 @@ Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, Constant *Op1, if (Opc == Instruction::And) { KnownBits Known0 = computeKnownBits(Op0, DL); KnownBits Known1 = computeKnownBits(Op1, DL); - if ((Known1.One | Known0.Zero).isAllOnesValue()) { + if ((Known1.One | Known0.Zero).isAllOnes()) { // All the bits of Op0 that the 'and' could be masking are already zero. return Op0; } - if ((Known0.One | Known1.Zero).isAllOnesValue()) { + if ((Known0.One | Known1.Zero).isAllOnes()) { // All the bits of Op1 that the 'and' could be masking are already zero. return Op1; } @@ -867,17 +810,10 @@ Constant *CastGEPIndices(Type *SrcElemTy, ArrayRef<Constant *> Ops, } /// Strip the pointer casts, but preserve the address space information. -Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) { +Constant *StripPtrCastKeepAS(Constant *Ptr) { assert(Ptr->getType()->isPointerTy() && "Not a pointer type"); auto *OldPtrTy = cast<PointerType>(Ptr->getType()); Ptr = cast<Constant>(Ptr->stripPointerCasts()); - if (ForLoadOperand) { - while (isa<GlobalAlias>(Ptr) && !cast<GlobalAlias>(Ptr)->isInterposable() && - !cast<GlobalAlias>(Ptr)->getBaseObject()->isInterposable()) { - Ptr = cast<GlobalAlias>(Ptr)->getAliasee(); - } - } - auto *NewPtrTy = cast<PointerType>(Ptr->getType()); // Preserve the address space number of the pointer. @@ -893,8 +829,7 @@ Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) { Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, ArrayRef<Constant *> Ops, const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool ForLoadOperand) { + const TargetLibraryInfo *TLI) { const GEPOperator *InnermostGEP = GEP; bool InBounds = GEP->isInBounds(); @@ -939,7 +874,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, DL.getIndexedOffsetInType( SrcElemTy, makeArrayRef((Value * const *)Ops.data() + 1, Ops.size() - 1))); - Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand); + Ptr = StripPtrCastKeepAS(Ptr); // If this is a GEP of a GEP, fold it all into a single GEP. while (auto *GEP = dyn_cast<GEPOperator>(Ptr)) { @@ -961,7 +896,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, Ptr = cast<Constant>(GEP->getOperand(0)); SrcElemTy = GEP->getSourceElementType(); Offset += APInt(BitWidth, DL.getIndexedOffsetInType(SrcElemTy, NestedOps)); - Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand); + Ptr = StripPtrCastKeepAS(Ptr); } // If the base value for this address is a literal integer value, fold the @@ -985,72 +920,41 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, // we eliminate over-indexing of the notional static type array bounds. // This makes it easy to determine if the getelementptr is "inbounds". // Also, this helps GlobalOpt do SROA on GlobalVariables. - SmallVector<Constant *, 32> NewIdxs; - Type *Ty = PTy; - SrcElemTy = PTy->getElementType(); - do { - if (!Ty->isStructTy()) { - if (Ty->isPointerTy()) { - // The only pointer indexing we'll do is on the first index of the GEP. - if (!NewIdxs.empty()) - break; + // For GEPs of GlobalValues, use the value type even for opaque pointers. + // Otherwise use an i8 GEP. + if (auto *GV = dyn_cast<GlobalValue>(Ptr)) + SrcElemTy = GV->getValueType(); + else if (!PTy->isOpaque()) + SrcElemTy = PTy->getElementType(); + else + SrcElemTy = Type::getInt8Ty(Ptr->getContext()); - Ty = SrcElemTy; + if (!SrcElemTy->isSized()) + return nullptr; - // Only handle pointers to sized types, not pointers to functions. - if (!Ty->isSized()) - return nullptr; - } else { - Type *NextTy = GetElementPtrInst::getTypeAtIndex(Ty, (uint64_t)0); - if (!NextTy) - break; - Ty = NextTy; - } + Type *ElemTy = SrcElemTy; + SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset); + if (Offset != 0) + return nullptr; - // Determine which element of the array the offset points into. - APInt ElemSize(BitWidth, DL.getTypeAllocSize(Ty)); - if (ElemSize == 0) { - // The element size is 0. This may be [0 x Ty]*, so just use a zero - // index for this level and proceed to the next level to see if it can - // accommodate the offset. - NewIdxs.push_back(ConstantInt::get(IntIdxTy, 0)); - } else { - // The element size is non-zero divide the offset by the element - // size (rounding down), to compute the index at this level. - bool Overflow; - APInt NewIdx = Offset.sdiv_ov(ElemSize, Overflow); - if (Overflow) - break; - Offset -= NewIdx * ElemSize; - NewIdxs.push_back(ConstantInt::get(IntIdxTy, NewIdx)); - } - } else { - auto *STy = cast<StructType>(Ty); - // If we end up with an offset that isn't valid for this struct type, we - // can't re-form this GEP in a regular form, so bail out. The pointer - // operand likely went through casts that are necessary to make the GEP - // sensible. - const StructLayout &SL = *DL.getStructLayout(STy); - if (Offset.isNegative() || Offset.uge(SL.getSizeInBytes())) - break; + // Try to add additional zero indices to reach the desired result element + // type. + // TODO: Should we avoid extra zero indices if ResElemTy can't be reached and + // we'll have to insert a bitcast anyway? + while (ElemTy != ResElemTy) { + Type *NextTy = GetElementPtrInst::getTypeAtIndex(ElemTy, (uint64_t)0); + if (!NextTy) + break; - // Determine which field of the struct the offset points into. The - // getZExtValue is fine as we've already ensured that the offset is - // within the range representable by the StructLayout API. - unsigned ElIdx = SL.getElementContainingOffset(Offset.getZExtValue()); - NewIdxs.push_back(ConstantInt::get(Type::getInt32Ty(Ty->getContext()), - ElIdx)); - Offset -= APInt(BitWidth, SL.getElementOffset(ElIdx)); - Ty = STy->getTypeAtIndex(ElIdx); - } - } while (Ty != ResElemTy); + Indices.push_back(APInt::getZero(isa<StructType>(ElemTy) ? 32 : BitWidth)); + ElemTy = NextTy; + } - // If we haven't used up the entire offset by descending the static - // type, then the offset is pointing into the middle of an indivisible - // member, so we can't simplify it. - if (Offset != 0) - return nullptr; + SmallVector<Constant *, 32> NewIdxs; + for (const APInt &Index : Indices) + NewIdxs.push_back(ConstantInt::get( + Type::getIntNTy(Ptr->getContext(), Index.getBitWidth()), Index)); // Preserve the inrange index from the innermost GEP if possible. We must // have calculated the same indices up to and including the inrange index. @@ -1067,8 +971,9 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, // Create a GEP. Constant *C = ConstantExpr::getGetElementPtr(SrcElemTy, Ptr, NewIdxs, InBounds, InRangeIndex); - assert(C->getType()->getPointerElementType() == Ty && - "Computed GetElementPtr has unexpected type!"); + assert( + cast<PointerType>(C->getType())->isOpaqueOrPointeeTypeMatches(ElemTy) && + "Computed GetElementPtr has unexpected type!"); // If we ended up indexing a member with a type that doesn't match // the type of what the original indices indexed, add a cast. @@ -1099,8 +1004,7 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode, return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL); if (auto *GEP = dyn_cast<GEPOperator>(InstOrCE)) { - if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI, - /*ForLoadOperand*/ false)) + if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI)) return C; return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0], @@ -1375,21 +1279,31 @@ Constant *llvm::ConstantFoldCastOperand(unsigned Opcode, Constant *C, default: llvm_unreachable("Missing case"); case Instruction::PtrToInt: - // If the input is a inttoptr, eliminate the pair. This requires knowing - // the width of a pointer, so it can't be done in ConstantExpr::getCast. if (auto *CE = dyn_cast<ConstantExpr>(C)) { + Constant *FoldedValue = nullptr; + // If the input is a inttoptr, eliminate the pair. This requires knowing + // the width of a pointer, so it can't be done in ConstantExpr::getCast. if (CE->getOpcode() == Instruction::IntToPtr) { - Constant *Input = CE->getOperand(0); - unsigned InWidth = Input->getType()->getScalarSizeInBits(); - unsigned PtrWidth = DL.getPointerTypeSizeInBits(CE->getType()); - if (PtrWidth < InWidth) { - Constant *Mask = - ConstantInt::get(CE->getContext(), - APInt::getLowBitsSet(InWidth, PtrWidth)); - Input = ConstantExpr::getAnd(Input, Mask); + // zext/trunc the inttoptr to pointer size. + FoldedValue = ConstantExpr::getIntegerCast( + CE->getOperand(0), DL.getIntPtrType(CE->getType()), + /*IsSigned=*/false); + } else if (auto *GEP = dyn_cast<GEPOperator>(CE)) { + // If we have GEP, we can perform the following folds: + // (ptrtoint (gep null, x)) -> x + // (ptrtoint (gep (gep null, x), y) -> x + y, etc. + unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType()); + APInt BaseOffset(BitWidth, 0); + auto *Base = cast<Constant>(GEP->stripAndAccumulateConstantOffsets( + DL, BaseOffset, /*AllowNonInbounds=*/true)); + if (Base->isNullValue()) { + FoldedValue = ConstantInt::get(CE->getContext(), BaseOffset); } - // Do a zext or trunc to get to the dest size. - return ConstantExpr::getIntegerCast(Input, DestTy, false); + } + if (FoldedValue) { + // Do a zext or trunc to get to the ptrtoint dest size. + return ConstantExpr::getIntegerCast(FoldedValue, DestTy, + /*IsSigned=*/false); } } return ConstantExpr::getCast(Opcode, C, DestTy); @@ -1446,19 +1360,6 @@ Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C, return ConstantFoldLoadThroughBitcast(C, Ty, DL); } -Constant * -llvm::ConstantFoldLoadThroughGEPIndices(Constant *C, - ArrayRef<Constant *> Indices) { - // Loop over all of the operands, tracking down which value we are - // addressing. - for (Constant *Index : Indices) { - C = C->getAggregateElement(Index); - if (!C) - return nullptr; - } - return C; -} - //===----------------------------------------------------------------------===// // Constant Folding for Calls // @@ -1879,7 +1780,7 @@ static bool mayFoldConstrained(ConstrainedFPIntrinsic *CI, // know that its evaluation does not raise exceptions, so side effect // is absent. To allow removing the call, mark it as not accessing memory. if (EB && *EB != fp::ExceptionBehavior::ebIgnore) - CI->addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone); + CI->addFnAttr(Attribute::ReadNone); return true; } @@ -2112,7 +2013,7 @@ static Constant *ConstantFoldScalarCall1(StringRef Name, /// the host native double versions. Float versions are not called /// directly but for all these it is true (float)(f((double)arg)) == /// f(arg). Long double not supported yet. - APFloat APF = Op->getValueAPF(); + const APFloat &APF = Op->getValueAPF(); switch (IntrinsicID) { default: break; @@ -2163,7 +2064,9 @@ static Constant *ConstantFoldScalarCall1(StringRef Name, return nullptr; LibFunc Func = NotLibFunc; - TLI->getLibFunc(Name, Func); + if (!TLI->getLibFunc(Name, Func)) + return nullptr; + switch (Func) { default: break; @@ -2416,12 +2319,12 @@ static Constant *ConstantFoldScalarCall2(StringRef Name, if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) { if (!Ty->isFloatingPointTy()) return nullptr; - APFloat Op1V = Op1->getValueAPF(); + const APFloat &Op1V = Op1->getValueAPF(); if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) { if (Op2->getType() != Op1->getType()) return nullptr; - APFloat Op2V = Op2->getValueAPF(); + const APFloat &Op2V = Op2->getValueAPF(); if (const auto *ConstrIntr = dyn_cast<ConstrainedFPIntrinsic>(Call)) { RoundingMode RM = getEvaluationRoundingMode(ConstrIntr); @@ -2487,7 +2390,9 @@ static Constant *ConstantFoldScalarCall2(StringRef Name, return nullptr; LibFunc Func = NotLibFunc; - TLI->getLibFunc(Name, Func); + if (!TLI->getLibFunc(Name, Func)) + return nullptr; + switch (Func) { default: break; @@ -2671,7 +2576,7 @@ static Constant *ConstantFoldScalarCall2(StringRef Name, assert(C1 && "Must be constant int"); // cttz(0, 1) and ctlz(0, 1) are undef. - if (C1->isOneValue() && (!C0 || C0->isNullValue())) + if (C1->isOne() && (!C0 || C0->isZero())) return UndefValue::get(Ty); if (!C0) return Constant::getNullValue(Ty); @@ -2683,11 +2588,11 @@ static Constant *ConstantFoldScalarCall2(StringRef Name, case Intrinsic::abs: // Undef or minimum val operand with poison min --> undef assert(C1 && "Must be constant int"); - if (C1->isOneValue() && (!C0 || C0->isMinSignedValue())) + if (C1->isOne() && (!C0 || C0->isMinSignedValue())) return UndefValue::get(Ty); // Undef operand with no poison min --> 0 (sign bit must be clear) - if (C1->isNullValue() && !C0) + if (C1->isZero() && !C0) return Constant::getNullValue(Ty); return ConstantInt::get(Ty, C0->abs()); @@ -3191,7 +3096,7 @@ bool llvm::isMathLibCallNoop(const CallBase *Call, if (!TLI || !TLI->getLibFunc(*F, Func)) return false; - if (Call->getNumArgOperands() == 1) { + if (Call->arg_size() == 1) { if (ConstantFP *OpC = dyn_cast<ConstantFP>(Call->getArgOperand(0))) { const APFloat &Op = OpC->getValueAPF(); switch (Func) { @@ -3280,7 +3185,7 @@ bool llvm::isMathLibCallNoop(const CallBase *Call, } } - if (Call->getNumArgOperands() == 2) { + if (Call->arg_size() == 2) { ConstantFP *Op0C = dyn_cast<ConstantFP>(Call->getArgOperand(0)); ConstantFP *Op1C = dyn_cast<ConstantFP>(Call->getArgOperand(1)); if (Op0C && Op1C) { diff --git a/llvm/lib/Analysis/CostModel.cpp b/llvm/lib/Analysis/CostModel.cpp index 83b7d5cbfc3e..f407ec0d017a 100644 --- a/llvm/lib/Analysis/CostModel.cpp +++ b/llvm/lib/Analysis/CostModel.cpp @@ -16,10 +16,12 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Analysis/CostModel.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" @@ -113,3 +115,23 @@ void CostModelAnalysis::print(raw_ostream &OS, const Module*) const { } } } + +PreservedAnalyses CostModelPrinterPass::run(Function &F, + FunctionAnalysisManager &AM) { + auto &TTI = AM.getResult<TargetIRAnalysis>(F); + OS << "Cost Model for function '" << F.getName() << "'\n"; + for (BasicBlock &B : F) { + for (Instruction &Inst : B) { + // TODO: Use a pass parameter instead of cl::opt CostKind to determine + // which cost kind to print. + InstructionCost Cost = TTI.getInstructionCost(&Inst, CostKind); + if (auto CostVal = Cost.getValue()) + OS << "Cost Model: Found an estimated cost of " << *CostVal; + else + OS << "Cost Model: Invalid cost"; + + OS << " for instruction: " << Inst << "\n"; + } + } + return PreservedAnalyses::all(); +} diff --git a/llvm/lib/Analysis/Delinearization.cpp b/llvm/lib/Analysis/Delinearization.cpp index 448e970e9bcc..670532c6d9a8 100644 --- a/llvm/lib/Analysis/Delinearization.cpp +++ b/llvm/lib/Analysis/Delinearization.cpp @@ -17,6 +17,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionDivision.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" @@ -36,6 +37,492 @@ using namespace llvm; #define DL_NAME "delinearize" #define DEBUG_TYPE DL_NAME +// Return true when S contains at least an undef value. +static inline bool containsUndefs(const SCEV *S) { + return SCEVExprContains(S, [](const SCEV *S) { + if (const auto *SU = dyn_cast<SCEVUnknown>(S)) + return isa<UndefValue>(SU->getValue()); + return false; + }); +} + +namespace { + +// Collect all steps of SCEV expressions. +struct SCEVCollectStrides { + ScalarEvolution &SE; + SmallVectorImpl<const SCEV *> &Strides; + + SCEVCollectStrides(ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &S) + : SE(SE), Strides(S) {} + + bool follow(const SCEV *S) { + if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) + Strides.push_back(AR->getStepRecurrence(SE)); + return true; + } + + bool isDone() const { return false; } +}; + +// Collect all SCEVUnknown and SCEVMulExpr expressions. +struct SCEVCollectTerms { + SmallVectorImpl<const SCEV *> &Terms; + + SCEVCollectTerms(SmallVectorImpl<const SCEV *> &T) : Terms(T) {} + + bool follow(const SCEV *S) { + if (isa<SCEVUnknown>(S) || isa<SCEVMulExpr>(S) || + isa<SCEVSignExtendExpr>(S)) { + if (!containsUndefs(S)) + Terms.push_back(S); + + // Stop recursion: once we collected a term, do not walk its operands. + return false; + } + + // Keep looking. + return true; + } + + bool isDone() const { return false; } +}; + +// Check if a SCEV contains an AddRecExpr. +struct SCEVHasAddRec { + bool &ContainsAddRec; + + SCEVHasAddRec(bool &ContainsAddRec) : ContainsAddRec(ContainsAddRec) { + ContainsAddRec = false; + } + + bool follow(const SCEV *S) { + if (isa<SCEVAddRecExpr>(S)) { + ContainsAddRec = true; + + // Stop recursion: once we collected a term, do not walk its operands. + return false; + } + + // Keep looking. + return true; + } + + bool isDone() const { return false; } +}; + +// Find factors that are multiplied with an expression that (possibly as a +// subexpression) contains an AddRecExpr. In the expression: +// +// 8 * (100 + %p * %q * (%a + {0, +, 1}_loop)) +// +// "%p * %q" are factors multiplied by the expression "(%a + {0, +, 1}_loop)" +// that contains the AddRec {0, +, 1}_loop. %p * %q are likely to be array size +// parameters as they form a product with an induction variable. +// +// This collector expects all array size parameters to be in the same MulExpr. +// It might be necessary to later add support for collecting parameters that are +// spread over different nested MulExpr. +struct SCEVCollectAddRecMultiplies { + SmallVectorImpl<const SCEV *> &Terms; + ScalarEvolution &SE; + + SCEVCollectAddRecMultiplies(SmallVectorImpl<const SCEV *> &T, + ScalarEvolution &SE) + : Terms(T), SE(SE) {} + + bool follow(const SCEV *S) { + if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) { + bool HasAddRec = false; + SmallVector<const SCEV *, 0> Operands; + for (auto Op : Mul->operands()) { + const SCEVUnknown *Unknown = dyn_cast<SCEVUnknown>(Op); + if (Unknown && !isa<CallInst>(Unknown->getValue())) { + Operands.push_back(Op); + } else if (Unknown) { + HasAddRec = true; + } else { + bool ContainsAddRec = false; + SCEVHasAddRec ContiansAddRec(ContainsAddRec); + visitAll(Op, ContiansAddRec); + HasAddRec |= ContainsAddRec; + } + } + if (Operands.size() == 0) + return true; + + if (!HasAddRec) + return false; + + Terms.push_back(SE.getMulExpr(Operands)); + // Stop recursion: once we collected a term, do not walk its operands. + return false; + } + + // Keep looking. + return true; + } + + bool isDone() const { return false; } +}; + +} // end anonymous namespace + +/// Find parametric terms in this SCEVAddRecExpr. We first for parameters in +/// two places: +/// 1) The strides of AddRec expressions. +/// 2) Unknowns that are multiplied with AddRec expressions. +void llvm::collectParametricTerms(ScalarEvolution &SE, const SCEV *Expr, + SmallVectorImpl<const SCEV *> &Terms) { + SmallVector<const SCEV *, 4> Strides; + SCEVCollectStrides StrideCollector(SE, Strides); + visitAll(Expr, StrideCollector); + + LLVM_DEBUG({ + dbgs() << "Strides:\n"; + for (const SCEV *S : Strides) + dbgs() << *S << "\n"; + }); + + for (const SCEV *S : Strides) { + SCEVCollectTerms TermCollector(Terms); + visitAll(S, TermCollector); + } + + LLVM_DEBUG({ + dbgs() << "Terms:\n"; + for (const SCEV *T : Terms) + dbgs() << *T << "\n"; + }); + + SCEVCollectAddRecMultiplies MulCollector(Terms, SE); + visitAll(Expr, MulCollector); +} + +static bool findArrayDimensionsRec(ScalarEvolution &SE, + SmallVectorImpl<const SCEV *> &Terms, + SmallVectorImpl<const SCEV *> &Sizes) { + int Last = Terms.size() - 1; + const SCEV *Step = Terms[Last]; + + // End of recursion. + if (Last == 0) { + if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) { + SmallVector<const SCEV *, 2> Qs; + for (const SCEV *Op : M->operands()) + if (!isa<SCEVConstant>(Op)) + Qs.push_back(Op); + + Step = SE.getMulExpr(Qs); + } + + Sizes.push_back(Step); + return true; + } + + for (const SCEV *&Term : Terms) { + // Normalize the terms before the next call to findArrayDimensionsRec. + const SCEV *Q, *R; + SCEVDivision::divide(SE, Term, Step, &Q, &R); + + // Bail out when GCD does not evenly divide one of the terms. + if (!R->isZero()) + return false; + + Term = Q; + } + + // Remove all SCEVConstants. + erase_if(Terms, [](const SCEV *E) { return isa<SCEVConstant>(E); }); + + if (Terms.size() > 0) + if (!findArrayDimensionsRec(SE, Terms, Sizes)) + return false; + + Sizes.push_back(Step); + return true; +} + +// Returns true when one of the SCEVs of Terms contains a SCEVUnknown parameter. +static inline bool containsParameters(SmallVectorImpl<const SCEV *> &Terms) { + for (const SCEV *T : Terms) + if (SCEVExprContains(T, [](const SCEV *S) { return isa<SCEVUnknown>(S); })) + return true; + + return false; +} + +// Return the number of product terms in S. +static inline int numberOfTerms(const SCEV *S) { + if (const SCEVMulExpr *Expr = dyn_cast<SCEVMulExpr>(S)) + return Expr->getNumOperands(); + return 1; +} + +static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) { + if (isa<SCEVConstant>(T)) + return nullptr; + + if (isa<SCEVUnknown>(T)) + return T; + + if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) { + SmallVector<const SCEV *, 2> Factors; + for (const SCEV *Op : M->operands()) + if (!isa<SCEVConstant>(Op)) + Factors.push_back(Op); + + return SE.getMulExpr(Factors); + } + + return T; +} + +void llvm::findArrayDimensions(ScalarEvolution &SE, + SmallVectorImpl<const SCEV *> &Terms, + SmallVectorImpl<const SCEV *> &Sizes, + const SCEV *ElementSize) { + if (Terms.size() < 1 || !ElementSize) + return; + + // Early return when Terms do not contain parameters: we do not delinearize + // non parametric SCEVs. + if (!containsParameters(Terms)) + return; + + LLVM_DEBUG({ + dbgs() << "Terms:\n"; + for (const SCEV *T : Terms) + dbgs() << *T << "\n"; + }); + + // Remove duplicates. + array_pod_sort(Terms.begin(), Terms.end()); + Terms.erase(std::unique(Terms.begin(), Terms.end()), Terms.end()); + + // Put larger terms first. + llvm::sort(Terms, [](const SCEV *LHS, const SCEV *RHS) { + return numberOfTerms(LHS) > numberOfTerms(RHS); + }); + + // Try to divide all terms by the element size. If term is not divisible by + // element size, proceed with the original term. + for (const SCEV *&Term : Terms) { + const SCEV *Q, *R; + SCEVDivision::divide(SE, Term, ElementSize, &Q, &R); + if (!Q->isZero()) + Term = Q; + } + + SmallVector<const SCEV *, 4> NewTerms; + + // Remove constant factors. + for (const SCEV *T : Terms) + if (const SCEV *NewT = removeConstantFactors(SE, T)) + NewTerms.push_back(NewT); + + LLVM_DEBUG({ + dbgs() << "Terms after sorting:\n"; + for (const SCEV *T : NewTerms) + dbgs() << *T << "\n"; + }); + + if (NewTerms.empty() || !findArrayDimensionsRec(SE, NewTerms, Sizes)) { + Sizes.clear(); + return; + } + + // The last element to be pushed into Sizes is the size of an element. + Sizes.push_back(ElementSize); + + LLVM_DEBUG({ + dbgs() << "Sizes:\n"; + for (const SCEV *S : Sizes) + dbgs() << *S << "\n"; + }); +} + +void llvm::computeAccessFunctions(ScalarEvolution &SE, const SCEV *Expr, + SmallVectorImpl<const SCEV *> &Subscripts, + SmallVectorImpl<const SCEV *> &Sizes) { + // Early exit in case this SCEV is not an affine multivariate function. + if (Sizes.empty()) + return; + + if (auto *AR = dyn_cast<SCEVAddRecExpr>(Expr)) + if (!AR->isAffine()) + return; + + const SCEV *Res = Expr; + int Last = Sizes.size() - 1; + for (int i = Last; i >= 0; i--) { + const SCEV *Q, *R; + SCEVDivision::divide(SE, Res, Sizes[i], &Q, &R); + + LLVM_DEBUG({ + dbgs() << "Res: " << *Res << "\n"; + dbgs() << "Sizes[i]: " << *Sizes[i] << "\n"; + dbgs() << "Res divided by Sizes[i]:\n"; + dbgs() << "Quotient: " << *Q << "\n"; + dbgs() << "Remainder: " << *R << "\n"; + }); + + Res = Q; + + // Do not record the last subscript corresponding to the size of elements in + // the array. + if (i == Last) { + + // Bail out if the byte offset is non-zero. + if (!R->isZero()) { + Subscripts.clear(); + Sizes.clear(); + return; + } + + continue; + } + + // Record the access function for the current subscript. + Subscripts.push_back(R); + } + + // Also push in last position the remainder of the last division: it will be + // the access function of the innermost dimension. + Subscripts.push_back(Res); + + std::reverse(Subscripts.begin(), Subscripts.end()); + + LLVM_DEBUG({ + dbgs() << "Subscripts:\n"; + for (const SCEV *S : Subscripts) + dbgs() << *S << "\n"; + }); +} + +/// Splits the SCEV into two vectors of SCEVs representing the subscripts and +/// sizes of an array access. Returns the remainder of the delinearization that +/// is the offset start of the array. The SCEV->delinearize algorithm computes +/// the multiples of SCEV coefficients: that is a pattern matching of sub +/// expressions in the stride and base of a SCEV corresponding to the +/// computation of a GCD (greatest common divisor) of base and stride. When +/// SCEV->delinearize fails, it returns the SCEV unchanged. +/// +/// For example: when analyzing the memory access A[i][j][k] in this loop nest +/// +/// void foo(long n, long m, long o, double A[n][m][o]) { +/// +/// for (long i = 0; i < n; i++) +/// for (long j = 0; j < m; j++) +/// for (long k = 0; k < o; k++) +/// A[i][j][k] = 1.0; +/// } +/// +/// the delinearization input is the following AddRec SCEV: +/// +/// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k> +/// +/// From this SCEV, we are able to say that the base offset of the access is %A +/// because it appears as an offset that does not divide any of the strides in +/// the loops: +/// +/// CHECK: Base offset: %A +/// +/// and then SCEV->delinearize determines the size of some of the dimensions of +/// the array as these are the multiples by which the strides are happening: +/// +/// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double) +/// bytes. +/// +/// Note that the outermost dimension remains of UnknownSize because there are +/// no strides that would help identifying the size of the last dimension: when +/// the array has been statically allocated, one could compute the size of that +/// dimension by dividing the overall size of the array by the size of the known +/// dimensions: %m * %o * 8. +/// +/// Finally delinearize provides the access functions for the array reference +/// that does correspond to A[i][j][k] of the above C testcase: +/// +/// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>] +/// +/// The testcases are checking the output of a function pass: +/// DelinearizationPass that walks through all loads and stores of a function +/// asking for the SCEV of the memory access with respect to all enclosing +/// loops, calling SCEV->delinearize on that and printing the results. +void llvm::delinearize(ScalarEvolution &SE, const SCEV *Expr, + SmallVectorImpl<const SCEV *> &Subscripts, + SmallVectorImpl<const SCEV *> &Sizes, + const SCEV *ElementSize) { + // First step: collect parametric terms. + SmallVector<const SCEV *, 4> Terms; + collectParametricTerms(SE, Expr, Terms); + + if (Terms.empty()) + return; + + // Second step: find subscript sizes. + findArrayDimensions(SE, Terms, Sizes, ElementSize); + + if (Sizes.empty()) + return; + + // Third step: compute the access functions for each subscript. + computeAccessFunctions(SE, Expr, Subscripts, Sizes); + + if (Subscripts.empty()) + return; + + LLVM_DEBUG({ + dbgs() << "succeeded to delinearize " << *Expr << "\n"; + dbgs() << "ArrayDecl[UnknownSize]"; + for (const SCEV *S : Sizes) + dbgs() << "[" << *S << "]"; + + dbgs() << "\nArrayRef"; + for (const SCEV *S : Subscripts) + dbgs() << "[" << *S << "]"; + dbgs() << "\n"; + }); +} + +bool llvm::getIndexExpressionsFromGEP(ScalarEvolution &SE, + const GetElementPtrInst *GEP, + SmallVectorImpl<const SCEV *> &Subscripts, + SmallVectorImpl<int> &Sizes) { + assert(Subscripts.empty() && Sizes.empty() && + "Expected output lists to be empty on entry to this function."); + assert(GEP && "getIndexExpressionsFromGEP called with a null GEP"); + Type *Ty = nullptr; + bool DroppedFirstDim = false; + for (unsigned i = 1; i < GEP->getNumOperands(); i++) { + const SCEV *Expr = SE.getSCEV(GEP->getOperand(i)); + if (i == 1) { + Ty = GEP->getSourceElementType(); + if (auto *Const = dyn_cast<SCEVConstant>(Expr)) + if (Const->getValue()->isZero()) { + DroppedFirstDim = true; + continue; + } + Subscripts.push_back(Expr); + continue; + } + + auto *ArrayTy = dyn_cast<ArrayType>(Ty); + if (!ArrayTy) { + Subscripts.clear(); + Sizes.clear(); + return false; + } + + Subscripts.push_back(Expr); + if (!(DroppedFirstDim && i == 2)) + Sizes.push_back(ArrayTy->getNumElements()); + + Ty = ArrayTy->getElementType(); + } + return !Subscripts.empty(); +} + namespace { class Delinearization : public FunctionPass { @@ -84,7 +571,7 @@ void printDelinearization(raw_ostream &O, Function *F, LoopInfo *LI, O << "AccessFunction: " << *AccessFn << "\n"; SmallVector<const SCEV *, 3> Subscripts, Sizes; - SE->delinearize(AccessFn, Subscripts, Sizes, SE->getElementSize(&Inst)); + delinearize(*SE, AccessFn, Subscripts, Sizes, SE->getElementSize(&Inst)); if (Subscripts.size() == 0 || Sizes.size() == 0 || Subscripts.size() != Sizes.size()) { O << "failed to delinearize\n"; diff --git a/llvm/lib/Analysis/DemandedBits.cpp b/llvm/lib/Analysis/DemandedBits.cpp index ca6d58fac825..117b12fc0701 100644 --- a/llvm/lib/Analysis/DemandedBits.cpp +++ b/llvm/lib/Analysis/DemandedBits.cpp @@ -362,7 +362,7 @@ void DemandedBits::performAnalysis() { if (Instruction *J = dyn_cast<Instruction>(OI)) { Type *T = J->getType(); if (T->isIntOrIntVectorTy()) - AliveBits[J] = APInt::getAllOnesValue(T->getScalarSizeInBits()); + AliveBits[J] = APInt::getAllOnes(T->getScalarSizeInBits()); else Visited.insert(J); Worklist.insert(J); @@ -407,7 +407,7 @@ void DemandedBits::performAnalysis() { Type *T = OI->getType(); if (T->isIntOrIntVectorTy()) { unsigned BitWidth = T->getScalarSizeInBits(); - APInt AB = APInt::getAllOnesValue(BitWidth); + APInt AB = APInt::getAllOnes(BitWidth); if (InputIsKnownDead) { AB = APInt(BitWidth, 0); } else { @@ -417,7 +417,7 @@ void DemandedBits::performAnalysis() { Known, Known2, KnownBitsComputed); // Keep track of uses which have no demanded bits. - if (AB.isNullValue()) + if (AB.isZero()) DeadUses.insert(&OI); else DeadUses.erase(&OI); @@ -448,8 +448,7 @@ APInt DemandedBits::getDemandedBits(Instruction *I) { return Found->second; const DataLayout &DL = I->getModule()->getDataLayout(); - return APInt::getAllOnesValue( - DL.getTypeSizeInBits(I->getType()->getScalarType())); + return APInt::getAllOnes(DL.getTypeSizeInBits(I->getType()->getScalarType())); } APInt DemandedBits::getDemandedBits(Use *U) { @@ -461,7 +460,7 @@ APInt DemandedBits::getDemandedBits(Use *U) { // We only track integer uses, everything else produces a mask with all bits // set if (!T->isIntOrIntVectorTy()) - return APInt::getAllOnesValue(BitWidth); + return APInt::getAllOnes(BitWidth); if (isUseDead(U)) return APInt(BitWidth, 0); @@ -469,7 +468,7 @@ APInt DemandedBits::getDemandedBits(Use *U) { performAnalysis(); APInt AOut = getDemandedBits(UserI); - APInt AB = APInt::getAllOnesValue(BitWidth); + APInt AB = APInt::getAllOnes(BitWidth); KnownBits Known, Known2; bool KnownBitsComputed = false; @@ -504,7 +503,7 @@ bool DemandedBits::isUseDead(Use *U) { // is dead. These uses might not be explicitly present in the DeadUses map. if (UserI->getType()->isIntOrIntVectorTy()) { auto Found = AliveBits.find(UserI); - if (Found != AliveBits.end() && Found->second.isNullValue()) + if (Found != AliveBits.end() && Found->second.isZero()) return true; } diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp index 9564cfb2aa45..f827f74d5367 100644 --- a/llvm/lib/Analysis/DependenceAnalysis.cpp +++ b/llvm/lib/Analysis/DependenceAnalysis.cpp @@ -53,6 +53,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/Delinearization.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" @@ -119,6 +120,11 @@ static cl::opt<bool> DisableDelinearizationChecks( "dependence vectors for languages that allow the subscript of one " "dimension to underflow or overflow into another dimension.")); +static cl::opt<unsigned> MIVMaxLevelThreshold( + "da-miv-max-level-threshold", cl::init(7), cl::Hidden, cl::ZeroOrMore, + cl::desc("Maximum depth allowed for the recursive algorithm used to " + "explore MIV direction vectors.")); + //===----------------------------------------------------------------------===// // basics @@ -2319,7 +2325,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, LLVM_DEBUG(dbgs() << "starting gcd\n"); ++GCDapplications; unsigned BitWidth = SE->getTypeSizeInBits(Src->getType()); - APInt RunningGCD = APInt::getNullValue(BitWidth); + APInt RunningGCD = APInt::getZero(BitWidth); // Examine Src coefficients. // Compute running GCD and record source constant. @@ -2359,7 +2365,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, } const SCEV *DstConst = Coefficients; - APInt ExtraGCD = APInt::getNullValue(BitWidth); + APInt ExtraGCD = APInt::getZero(BitWidth); const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst); LLVM_DEBUG(dbgs() << " Delta = " << *Delta << "\n"); const SCEVConstant *Constant = dyn_cast<SCEVConstant>(Delta); @@ -2602,6 +2608,19 @@ unsigned DependenceInfo::exploreDirections(unsigned Level, CoefficientInfo *A, const SmallBitVector &Loops, unsigned &DepthExpanded, const SCEV *Delta) const { + // This algorithm has worst case complexity of O(3^n), where 'n' is the number + // of common loop levels. To avoid excessive compile-time, pessimize all the + // results and immediately return when the number of common levels is beyond + // the given threshold. + if (CommonLevels > MIVMaxLevelThreshold) { + LLVM_DEBUG(dbgs() << "Number of common levels exceeded the threshold. MIV " + "direction exploration is terminated.\n"); + for (unsigned K = 1; K <= CommonLevels; ++K) + if (Loops[K]) + Bound[K].DirSet = Dependence::DVEntry::ALL; + return 1; + } + if (Level > CommonLevels) { // record result LLVM_DEBUG(dbgs() << "\t["); @@ -3320,8 +3339,8 @@ bool DependenceInfo::tryDelinearizeFixedSize( return false; SmallVector<int, 4> SrcSizes, DstSizes; - SE->getIndexExpressionsFromGEP(SrcGEP, SrcSubscripts, SrcSizes); - SE->getIndexExpressionsFromGEP(DstGEP, DstSubscripts, DstSizes); + getIndexExpressionsFromGEP(*SE, SrcGEP, SrcSubscripts, SrcSizes); + getIndexExpressionsFromGEP(*SE, DstGEP, DstSubscripts, DstSizes); // Check that the two size arrays are non-empty and equal in length and // value. @@ -3421,16 +3440,16 @@ bool DependenceInfo::tryDelinearizeParametricSize( // First step: collect parametric terms in both array references. SmallVector<const SCEV *, 4> Terms; - SE->collectParametricTerms(SrcAR, Terms); - SE->collectParametricTerms(DstAR, Terms); + collectParametricTerms(*SE, SrcAR, Terms); + collectParametricTerms(*SE, DstAR, Terms); // Second step: find subscript sizes. SmallVector<const SCEV *, 4> Sizes; - SE->findArrayDimensions(Terms, Sizes, ElementSize); + findArrayDimensions(*SE, Terms, Sizes, ElementSize); // Third step: compute the access functions for each subscript. - SE->computeAccessFunctions(SrcAR, SrcSubscripts, Sizes); - SE->computeAccessFunctions(DstAR, DstSubscripts, Sizes); + computeAccessFunctions(*SE, SrcAR, SrcSubscripts, Sizes); + computeAccessFunctions(*SE, DstAR, DstSubscripts, Sizes); // Fail when there is only a subscript: that's a linearized access function. if (SrcSubscripts.size() < 2 || DstSubscripts.size() < 2 || diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp index ecfefa36918c..d87fa849d839 100644 --- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp +++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp @@ -1,9 +1,8 @@ //===- DevelopmentModeInlineAdvisor.cpp - runtime-loadable model runner --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -228,6 +227,8 @@ private: (*CallerSizeEstimateBefore + *CalleeSizeEstimateBefore); getAdvisor()->updateNativeSizeEstimate(Reward); log(Reward, /*Success=*/true); + } else { + log(NoReward, /*Success=*/true); } } @@ -377,7 +378,7 @@ void TrainingLogger::logInlineEvent(const InlineEvent &Event, void TrainingLogger::print() { std::error_code EC; raw_fd_ostream OutFile(LogFileName, EC); - L->print(OutFile); + L->flush(OutFile); } DevelopmentModeMLInlineAdvisor::DevelopmentModeMLInlineAdvisor( diff --git a/llvm/lib/Analysis/HeatUtils.cpp b/llvm/lib/Analysis/HeatUtils.cpp index a1a11be5fee3..0057de322cac 100644 --- a/llvm/lib/Analysis/HeatUtils.cpp +++ b/llvm/lib/Analysis/HeatUtils.cpp @@ -1,9 +1,8 @@ //===-- HeatUtils.cpp - Utility for printing heat colors --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/llvm/lib/Analysis/IRSimilarityIdentifier.cpp b/llvm/lib/Analysis/IRSimilarityIdentifier.cpp index a6298afb66f5..f22c6aa04f5e 100644 --- a/llvm/lib/Analysis/IRSimilarityIdentifier.cpp +++ b/llvm/lib/Analysis/IRSimilarityIdentifier.cpp @@ -23,13 +23,23 @@ using namespace llvm; using namespace IRSimilarity; +cl::opt<bool> + DisableBranches("no-ir-sim-branch-matching", cl::init(false), + cl::ReallyHidden, + cl::desc("disable similarity matching, and outlining, " + "across branches for debugging purposes.")); + IRInstructionData::IRInstructionData(Instruction &I, bool Legality, IRInstructionDataList &IDList) : Inst(&I), Legal(Legality), IDL(&IDList) { + initializeInstruction(); +} + +void IRInstructionData::initializeInstruction() { // We check for whether we have a comparison instruction. If it is, we // find the "less than" version of the predicate for consistency for // comparison instructions throught the program. - if (CmpInst *C = dyn_cast<CmpInst>(&I)) { + if (CmpInst *C = dyn_cast<CmpInst>(Inst)) { CmpInst::Predicate Predicate = predicateForConsistency(C); if (Predicate != C->getPredicate()) RevisedPredicate = Predicate; @@ -37,8 +47,8 @@ IRInstructionData::IRInstructionData(Instruction &I, bool Legality, // Here we collect the operands and their types for determining whether // the structure of the operand use matches between two different candidates. - for (Use &OI : I.operands()) { - if (isa<CmpInst>(I) && RevisedPredicate.hasValue()) { + for (Use &OI : Inst->operands()) { + if (isa<CmpInst>(Inst) && RevisedPredicate.hasValue()) { // If we have a CmpInst where the predicate is reversed, it means the // operands must be reversed as well. OperVals.insert(OperVals.begin(), OI.get()); @@ -49,6 +59,33 @@ IRInstructionData::IRInstructionData(Instruction &I, bool Legality, } } +IRInstructionData::IRInstructionData(IRInstructionDataList &IDList) + : Inst(nullptr), Legal(false), IDL(&IDList) {} + +void IRInstructionData::setBranchSuccessors( + DenseMap<BasicBlock *, unsigned> &BasicBlockToInteger) { + assert(isa<BranchInst>(Inst) && "Instruction must be branch"); + + BranchInst *BI = cast<BranchInst>(Inst); + DenseMap<BasicBlock *, unsigned>::iterator BBNumIt; + + BBNumIt = BasicBlockToInteger.find(BI->getParent()); + assert(BBNumIt != BasicBlockToInteger.end() && + "Could not find location for BasicBlock!"); + + int CurrentBlockNumber = static_cast<int>(BBNumIt->second); + + for (BasicBlock *Successor : BI->successors()) { + BBNumIt = BasicBlockToInteger.find(Successor); + assert(BBNumIt != BasicBlockToInteger.end() && + "Could not find number for BasicBlock!"); + int OtherBlockNumber = static_cast<int>(BBNumIt->second); + + int Relative = OtherBlockNumber - CurrentBlockNumber; + RelativeBlockLocations.push_back(Relative); + } +} + CmpInst::Predicate IRInstructionData::predicateForConsistency(CmpInst *CI) { switch (CI->getPredicate()) { case CmpInst::FCMP_OGT: @@ -143,6 +180,10 @@ bool IRSimilarity::isClose(const IRInstructionData &A, return false; } + if (isa<BranchInst>(A.Inst) && isa<BranchInst>(B.Inst) && + A.RelativeBlockLocations.size() != B.RelativeBlockLocations.size()) + return false; + return true; } @@ -156,10 +197,6 @@ void IRInstructionMapper::convertToUnsignedVec( std::vector<unsigned> IntegerMappingForBB; std::vector<IRInstructionData *> InstrListForBB; - HaveLegalRange = false; - CanCombineWithPrevInstr = false; - AddedIllegalLastTime = true; - for (BasicBlock::iterator Et = BB.end(); It != Et; ++It) { switch (InstClassifier.visit(*It)) { case InstrType::Legal: @@ -175,7 +212,8 @@ void IRInstructionMapper::convertToUnsignedVec( } if (HaveLegalRange) { - mapToIllegalUnsigned(It, IntegerMappingForBB, InstrListForBB, true); + if (AddedIllegalLastTime) + mapToIllegalUnsigned(It, IntegerMappingForBB, InstrListForBB, true); for (IRInstructionData *ID : InstrListForBB) this->IDL->push_back(*ID); llvm::append_range(InstrList, InstrListForBB); @@ -203,6 +241,9 @@ unsigned IRInstructionMapper::mapToLegalUnsigned( IRInstructionData *ID = allocateIRInstructionData(*It, true, *IDL); InstrListForBB.push_back(ID); + if (isa<BranchInst>(*It)) + ID->setBranchSuccessors(BasicBlockToInteger); + // Add to the instruction list bool WasInserted; DenseMap<IRInstructionData *, unsigned, IRInstructionDataTraits>::iterator @@ -235,6 +276,11 @@ IRInstructionMapper::allocateIRInstructionData(Instruction &I, bool Legality, return new (InstDataAllocator->Allocate()) IRInstructionData(I, Legality, IDL); } +IRInstructionData * +IRInstructionMapper::allocateIRInstructionData(IRInstructionDataList &IDL) { + return new (InstDataAllocator->Allocate()) IRInstructionData(IDL); +} + IRInstructionDataList * IRInstructionMapper::allocateIRInstructionDataList() { return new (IDLAllocator->Allocate()) IRInstructionDataList(); @@ -255,6 +301,8 @@ unsigned IRInstructionMapper::mapToIllegalUnsigned( IRInstructionData *ID = nullptr; if (!End) ID = allocateIRInstructionData(*It, false, *IDL); + else + ID = allocateIRInstructionData(*IDL); InstrListForBB.push_back(ID); // Remember that we added an illegal number last time. @@ -563,8 +611,50 @@ bool IRSimilarityCandidate::compareCommutativeOperandMapping( return true; } +bool IRSimilarityCandidate::checkRelativeLocations(RelativeLocMapping A, + RelativeLocMapping B) { + // Get the basic blocks the label refers to. + BasicBlock *ABB = static_cast<BasicBlock *>(A.OperVal); + BasicBlock *BBB = static_cast<BasicBlock *>(B.OperVal); + + // Get the basic blocks contained in each region. + DenseSet<BasicBlock *> BasicBlockA; + DenseSet<BasicBlock *> BasicBlockB; + A.IRSC.getBasicBlocks(BasicBlockA); + B.IRSC.getBasicBlocks(BasicBlockB); + + // Determine if the block is contained in the region. + bool AContained = BasicBlockA.contains(ABB); + bool BContained = BasicBlockB.contains(BBB); + + // Both blocks need to be contained in the region, or both need to be outside + // the reigon. + if (AContained != BContained) + return false; + + // If both are contained, then we need to make sure that the relative + // distance to the target blocks are the same. + if (AContained) + return A.RelativeLocation == B.RelativeLocation; + return true; +} + bool IRSimilarityCandidate::compareStructure(const IRSimilarityCandidate &A, const IRSimilarityCandidate &B) { + DenseMap<unsigned, DenseSet<unsigned>> MappingA; + DenseMap<unsigned, DenseSet<unsigned>> MappingB; + return IRSimilarityCandidate::compareStructure(A, B, MappingA, MappingB); +} + +typedef detail::zippy<detail::zip_shortest, SmallVector<int, 4> &, + SmallVector<int, 4> &, ArrayRef<Value *> &, + ArrayRef<Value *> &> + ZippedRelativeLocationsT; + +bool IRSimilarityCandidate::compareStructure( + const IRSimilarityCandidate &A, const IRSimilarityCandidate &B, + DenseMap<unsigned, DenseSet<unsigned>> &ValueNumberMappingA, + DenseMap<unsigned, DenseSet<unsigned>> &ValueNumberMappingB) { if (A.getLength() != B.getLength()) return false; @@ -574,15 +664,12 @@ bool IRSimilarityCandidate::compareStructure(const IRSimilarityCandidate &A, iterator ItA = A.begin(); iterator ItB = B.begin(); - // These sets create a create a mapping between the values in one candidate - // to values in the other candidate. If we create a set with one element, - // and that same element maps to the original element in the candidate - // we have a good mapping. - DenseMap<unsigned, DenseSet<unsigned>> ValueNumberMappingA; - DenseMap<unsigned, DenseSet<unsigned>> ValueNumberMappingB; + // These ValueNumber Mapping sets create a create a mapping between the values + // in one candidate to values in the other candidate. If we create a set with + // one element, and that same element maps to the original element in the + // candidate we have a good mapping. DenseMap<unsigned, DenseSet<unsigned>>::iterator ValueMappingIt; - bool WasInserted; // Iterate over the instructions contained in each candidate unsigned SectionLength = A.getStartIdx() + A.getLength(); @@ -605,6 +692,7 @@ bool IRSimilarityCandidate::compareStructure(const IRSimilarityCandidate &A, unsigned InstValA = A.ValueToNumber.find(IA)->second; unsigned InstValB = B.ValueToNumber.find(IB)->second; + bool WasInserted; // Ensure that the mappings for the instructions exists. std::tie(ValueMappingIt, WasInserted) = ValueNumberMappingA.insert( std::make_pair(InstValA, DenseSet<unsigned>({InstValB}))); @@ -632,6 +720,37 @@ bool IRSimilarityCandidate::compareStructure(const IRSimilarityCandidate &A, {A, OperValsA, ValueNumberMappingA}, {B, OperValsB, ValueNumberMappingB})) return false; + + // Here we check that between two corresponding instructions, + // when referring to a basic block in the same region, the + // relative locations are the same. And, that the instructions refer to + // basic blocks outside the region in the same corresponding locations. + + // We are able to make the assumption about blocks outside of the region + // since the target block labels are considered values and will follow the + // same number matching that we defined for the other instructions in the + // region. So, at this point, in each location we target a specific block + // outside the region, we are targeting a corresponding block in each + // analagous location in the region we are comparing to. + if (!(isa<BranchInst>(IA) && isa<BranchInst>(IB)) && + !(isa<PHINode>(IA) && isa<PHINode>(IB))) + continue; + + SmallVector<int, 4> &RelBlockLocsA = ItA->RelativeBlockLocations; + SmallVector<int, 4> &RelBlockLocsB = ItB->RelativeBlockLocations; + if (RelBlockLocsA.size() != RelBlockLocsB.size() && + OperValsA.size() != OperValsB.size()) + return false; + + ZippedRelativeLocationsT ZippedRelativeLocations = + zip(RelBlockLocsA, RelBlockLocsB, OperValsA, OperValsB); + if (any_of(ZippedRelativeLocations, + [&A, &B](std::tuple<int, int, Value *, Value *> R) { + return !checkRelativeLocations( + {A, std::get<0>(R), std::get<2>(R)}, + {B, std::get<1>(R), std::get<3>(R)}); + })) + return false; } return true; } @@ -657,6 +776,8 @@ void IRSimilarityIdentifier::populateMapper( std::vector<unsigned> IntegerMappingForModule; // Iterate over the functions in the module to map each Instruction in each // BasicBlock to an unsigned integer. + Mapper.initializeForBBs(M); + for (Function &F : M) { if (F.empty()) @@ -664,15 +785,18 @@ void IRSimilarityIdentifier::populateMapper( for (BasicBlock &BB : F) { - if (BB.sizeWithoutDebug() < 2) - continue; - // BB has potential to have similarity since it has a size greater than 2 // and can therefore match other regions greater than 2. Map it to a list // of unsigned integers. Mapper.convertToUnsignedVec(BB, InstrListForModule, IntegerMappingForModule); } + + BasicBlock::iterator It = F.begin()->end(); + Mapper.mapToIllegalUnsigned(It, IntegerMappingForModule, InstrListForModule, + true); + if (InstrListForModule.size() > 0) + Mapper.IDL->push_back(*InstrListForModule.back()); } // Insert the InstrListForModule at the end of the overall InstrList so that @@ -707,6 +831,8 @@ static void createCandidatesFromSuffixTree( std::vector<IRSimilarityCandidate> &CandsForRepSubstring) { unsigned StringLen = RS.Length; + if (StringLen < 2) + return; // Create an IRSimilarityCandidate for instance of this subsequence \p RS. for (const unsigned &StartIdx : RS.StartIndices) { @@ -739,6 +865,84 @@ static void createCandidatesFromSuffixTree( } } +void IRSimilarityCandidate::createCanonicalRelationFrom( + IRSimilarityCandidate &SourceCand, + DenseMap<unsigned, DenseSet<unsigned>> &ToSourceMapping, + DenseMap<unsigned, DenseSet<unsigned>> &FromSourceMapping) { + assert(SourceCand.CanonNumToNumber.size() != 0 && + "Base canonical relationship is empty!"); + assert(SourceCand.NumberToCanonNum.size() != 0 && + "Base canonical relationship is empty!"); + + assert(CanonNumToNumber.size() == 0 && "Canonical Relationship is non-empty"); + assert(NumberToCanonNum.size() == 0 && "Canonical Relationship is non-empty"); + + DenseSet<unsigned> UsedGVNs; + // Iterate over the mappings provided from this candidate to SourceCand. We + // are then able to map the GVN in this candidate to the same canonical number + // given to the corresponding GVN in SourceCand. + for (std::pair<unsigned, DenseSet<unsigned>> &GVNMapping : ToSourceMapping) { + unsigned SourceGVN = GVNMapping.first; + + assert(GVNMapping.second.size() != 0 && "Possible GVNs is 0!"); + + unsigned ResultGVN; + // We need special handling if we have more than one potential value. This + // means that there are at least two GVNs that could correspond to this GVN. + // This could lead to potential swapping later on, so we make a decision + // here to ensure a one-to-one mapping. + if (GVNMapping.second.size() > 1) { + bool Found = false; + for (unsigned Val : GVNMapping.second) { + // We make sure the target value number hasn't already been reserved. + if (UsedGVNs.contains(Val)) + continue; + + // We make sure that the opposite mapping is still consistent. + DenseMap<unsigned, DenseSet<unsigned>>::iterator It = + FromSourceMapping.find(Val); + + if (!It->second.contains(SourceGVN)) + continue; + + // We pick the first item that satisfies these conditions. + Found = true; + ResultGVN = Val; + break; + } + + assert(Found && "Could not find matching value for source GVN"); + (void)Found; + + } else + ResultGVN = *GVNMapping.second.begin(); + + // Whatever GVN is found, we mark it as used. + UsedGVNs.insert(ResultGVN); + + unsigned CanonNum = *SourceCand.getCanonicalNum(ResultGVN); + CanonNumToNumber.insert(std::make_pair(CanonNum, SourceGVN)); + NumberToCanonNum.insert(std::make_pair(SourceGVN, CanonNum)); + } +} + +void IRSimilarityCandidate::createCanonicalMappingFor( + IRSimilarityCandidate &CurrCand) { + assert(CurrCand.CanonNumToNumber.size() == 0 && + "Canonical Relationship is non-empty"); + assert(CurrCand.NumberToCanonNum.size() == 0 && + "Canonical Relationship is non-empty"); + + unsigned CanonNum = 0; + // Iterate over the value numbers found, the order does not matter in this + // case. + for (std::pair<unsigned, Value *> &NumToVal : CurrCand.NumberToValue) { + CurrCand.NumberToCanonNum.insert(std::make_pair(NumToVal.first, CanonNum)); + CurrCand.CanonNumToNumber.insert(std::make_pair(CanonNum, NumToVal.first)); + CanonNum++; + } +} + /// From the list of IRSimilarityCandidates, perform a comparison between each /// IRSimilarityCandidate to determine if there are overlapping /// IRInstructionData, or if they do not have the same structure. @@ -774,6 +978,8 @@ static void findCandidateStructures( // Iterate over the candidates to determine its structural and overlapping // compatibility with other instructions + DenseMap<unsigned, DenseSet<unsigned>> ValueNumberMappingA; + DenseMap<unsigned, DenseSet<unsigned>> ValueNumberMappingB; for (CandIt = CandsForRepSubstring.begin(), CandEndIt = CandsForRepSubstring.end(); CandIt != CandEndIt; CandIt++) { @@ -792,9 +998,11 @@ static void findCandidateStructures( // Check if we already have a list of IRSimilarityCandidates for the current // structural group. Create one if one does not exist. CurrentGroupPair = StructuralGroups.find(OuterGroupNum); - if (CurrentGroupPair == StructuralGroups.end()) + if (CurrentGroupPair == StructuralGroups.end()) { + IRSimilarityCandidate::createCanonicalMappingFor(*CandIt); std::tie(CurrentGroupPair, Inserted) = StructuralGroups.insert( std::make_pair(OuterGroupNum, SimilarityGroup({*CandIt}))); + } // Iterate over the IRSimilarityCandidates following the current // IRSimilarityCandidate in the list to determine whether the two @@ -811,11 +1019,15 @@ static void findCandidateStructures( // Otherwise we determine if they have the same structure and add it to // vector if they match. - SameStructure = - IRSimilarityCandidate::compareStructure(*CandIt, *InnerCandIt); + ValueNumberMappingA.clear(); + ValueNumberMappingB.clear(); + SameStructure = IRSimilarityCandidate::compareStructure( + *CandIt, *InnerCandIt, ValueNumberMappingA, ValueNumberMappingB); if (!SameStructure) continue; + InnerCandIt->createCanonicalRelationFrom(*CandIt, ValueNumberMappingA, + ValueNumberMappingB); CandToGroup.insert(std::make_pair(&*InnerCandIt, OuterGroupNum)); CurrentGroupPair->second.push_back(*InnerCandIt); } @@ -862,6 +1074,7 @@ SimilarityGroupList &IRSimilarityIdentifier::findSimilarity( std::vector<IRInstructionData *> InstrList; std::vector<unsigned> IntegerMapping; + Mapper.InstClassifier.EnableBranches = this->EnableBranches; populateMapper(Modules, InstrList, IntegerMapping); findCandidates(InstrList, IntegerMapping); @@ -871,6 +1084,7 @@ SimilarityGroupList &IRSimilarityIdentifier::findSimilarity( SimilarityGroupList &IRSimilarityIdentifier::findSimilarity(Module &M) { resetSimilarityCandidates(); + Mapper.InstClassifier.EnableBranches = this->EnableBranches; std::vector<IRInstructionData *> InstrList; std::vector<unsigned> IntegerMapping; @@ -891,7 +1105,7 @@ IRSimilarityIdentifierWrapperPass::IRSimilarityIdentifierWrapperPass() } bool IRSimilarityIdentifierWrapperPass::doInitialization(Module &M) { - IRSI.reset(new IRSimilarityIdentifier()); + IRSI.reset(new IRSimilarityIdentifier(!DisableBranches)); return false; } @@ -907,9 +1121,9 @@ bool IRSimilarityIdentifierWrapperPass::runOnModule(Module &M) { AnalysisKey IRSimilarityAnalysis::Key; IRSimilarityIdentifier IRSimilarityAnalysis::run(Module &M, - ModuleAnalysisManager &) { + ModuleAnalysisManager &) { - auto IRSI = IRSimilarityIdentifier(); + auto IRSI = IRSimilarityIdentifier(!DisableBranches); IRSI.findSimilarity(M); return IRSI; } diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index fc6051b35efc..c4b7239b43ab 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -43,8 +43,8 @@ using namespace llvm::PatternMatch; bool RecurrenceDescriptor::areAllUsesIn(Instruction *I, SmallPtrSetImpl<Instruction *> &Set) { - for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use) - if (!Set.count(dyn_cast<Instruction>(*Use))) + for (const Use &Use : I->operands()) + if (!Set.count(dyn_cast<Instruction>(Use))) return false; return true; } @@ -62,6 +62,8 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) { case RecurKind::SMin: case RecurKind::UMax: case RecurKind::UMin: + case RecurKind::SelectICmp: + case RecurKind::SelectFCmp: return true; } return false; @@ -144,12 +146,9 @@ static std::pair<Type *, bool> computeRecurrenceType(Instruction *Exit, // meaning that we will use sext instructions instead of zext // instructions to restore the original type. IsSigned = true; - if (!Bits.isNegative()) - // If the value is not known to be negative, we don't known what the - // upper bit is, and therefore, we don't know what kind of extend we - // will need. In this case, just increase the bit width by one bit and - // use sext. - ++MaxBitWidth; + // Make sure at at least one sign bit is included in the result, so it + // will get properly sign-extended. + ++MaxBitWidth; } } if (!isPowerOf2_64(MaxBitWidth)) @@ -199,7 +198,10 @@ static bool checkOrderedReduction(RecurKind Kind, Instruction *ExactFPMathInst, if (Kind != RecurKind::FAdd) return false; - if (Exit->getOpcode() != Instruction::FAdd || Exit != ExactFPMathInst) + // Ensure the exit instruction is an FAdd, and that it only has one user + // other than the reduction PHI + if (Exit->getOpcode() != Instruction::FAdd || Exit->hasNUsesOrMore(3) || + Exit != ExactFPMathInst) return false; // The only pattern accepted is the one in which the reduction PHI @@ -272,7 +274,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, } else if (RecurrenceType->isIntegerTy()) { if (!isIntegerRecurrenceKind(Kind)) return false; - if (isArithmeticRecurrenceKind(Kind)) + if (!isMinMaxRecurrenceKind(Kind)) Start = lookThroughAnd(Phi, RecurrenceType, VisitedInsts, CastInsts); } else { // Pointer min/max may exist, but it is not supported as a reduction op. @@ -327,7 +329,8 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, // the starting value (the Phi or an AND instruction if the Phi has been // type-promoted). if (Cur != Start) { - ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, FuncFMF); + ReduxDesc = + isRecurrenceInstr(TheLoop, Phi, Cur, Kind, ReduxDesc, FuncFMF); if (!ReduxDesc.isRecurrence()) return false; // FIXME: FMF is allowed on phi, but propagation is not handled correctly. @@ -360,6 +363,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, // A reduction operation must only have one use of the reduction value. if (!IsAPhi && !IsASelect && !isMinMaxRecurrenceKind(Kind) && + !isSelectCmpRecurrenceKind(Kind) && hasMultipleUsesOf(Cur, VisitedInsts, 1)) return false; @@ -367,10 +371,10 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts)) return false; - if (isIntMinMaxRecurrenceKind(Kind) && + if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::SelectICmp) && (isa<ICmpInst>(Cur) || isa<SelectInst>(Cur))) ++NumCmpSelectPatternInst; - if (isFPMinMaxRecurrenceKind(Kind) && + if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::SelectFCmp) && (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur))) ++NumCmpSelectPatternInst; @@ -423,7 +427,9 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) && !isa<SelectInst>(UI)) || (!isConditionalRdxPattern(Kind, UI).isRecurrence() && - !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence()))) + !isSelectCmpPattern(TheLoop, Phi, UI, IgnoredVal) + .isRecurrence() && + !isMinMaxPattern(UI, Kind, IgnoredVal).isRecurrence()))) return false; // Remember that we completed the cycle. @@ -435,8 +441,13 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, } // This means we have seen one but not the other instruction of the - // pattern or more than just a select and cmp. - if (isMinMaxRecurrenceKind(Kind) && NumCmpSelectPatternInst != 2) + // pattern or more than just a select and cmp. Zero implies that we saw a + // llvm.min/max instrinsic, which is always OK. + if (isMinMaxRecurrenceKind(Kind) && NumCmpSelectPatternInst != 2 && + NumCmpSelectPatternInst != 0) + return false; + + if (isSelectCmpRecurrenceKind(Kind) && NumCmpSelectPatternInst != 1) return false; if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction) @@ -505,11 +516,70 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, return true; } +// We are looking for loops that do something like this: +// int r = 0; +// for (int i = 0; i < n; i++) { +// if (src[i] > 3) +// r = 3; +// } +// where the reduction value (r) only has two states, in this example 0 or 3. +// The generated LLVM IR for this type of loop will be like this: +// for.body: +// %r = phi i32 [ %spec.select, %for.body ], [ 0, %entry ] +// ... +// %cmp = icmp sgt i32 %5, 3 +// %spec.select = select i1 %cmp, i32 3, i32 %r +// ... +// In general we can support vectorization of loops where 'r' flips between +// any two non-constants, provided they are loop invariant. The only thing +// we actually care about at the end of the loop is whether or not any lane +// in the selected vector is different from the start value. The final +// across-vector reduction after the loop simply involves choosing the start +// value if nothing changed (0 in the example above) or the other selected +// value (3 in the example above). RecurrenceDescriptor::InstDesc -RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I, - const InstDesc &Prev) { - assert((isa<CmpInst>(I) || isa<SelectInst>(I)) && - "Expected a cmp or select instruction"); +RecurrenceDescriptor::isSelectCmpPattern(Loop *Loop, PHINode *OrigPhi, + Instruction *I, InstDesc &Prev) { + // We must handle the select(cmp(),x,y) as a single instruction. Advance to + // the select. + CmpInst::Predicate Pred; + if (match(I, m_OneUse(m_Cmp(Pred, m_Value(), m_Value())))) { + if (auto *Select = dyn_cast<SelectInst>(*I->user_begin())) + return InstDesc(Select, Prev.getRecKind()); + } + + // Only match select with single use cmp condition. + if (!match(I, m_Select(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())), m_Value(), + m_Value()))) + return InstDesc(false, I); + + SelectInst *SI = cast<SelectInst>(I); + Value *NonPhi = nullptr; + + if (OrigPhi == dyn_cast<PHINode>(SI->getTrueValue())) + NonPhi = SI->getFalseValue(); + else if (OrigPhi == dyn_cast<PHINode>(SI->getFalseValue())) + NonPhi = SI->getTrueValue(); + else + return InstDesc(false, I); + + // We are looking for selects of the form: + // select(cmp(), phi, loop_invariant) or + // select(cmp(), loop_invariant, phi) + if (!Loop->isLoopInvariant(NonPhi)) + return InstDesc(false, I); + + return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::SelectICmp + : RecurKind::SelectFCmp); +} + +RecurrenceDescriptor::InstDesc +RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind, + const InstDesc &Prev) { + assert((isa<CmpInst>(I) || isa<SelectInst>(I) || isa<CallInst>(I)) && + "Expected a cmp or select or call instruction"); + if (!isMinMaxRecurrenceKind(Kind)) + return InstDesc(false, I); // We must handle the select(cmp()) as a single instruction. Advance to the // select. @@ -519,28 +589,33 @@ RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I, return InstDesc(Select, Prev.getRecKind()); } - // Only match select with single use cmp condition. - if (!match(I, m_Select(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())), m_Value(), + // Only match select with single use cmp condition, or a min/max intrinsic. + if (!isa<IntrinsicInst>(I) && + !match(I, m_Select(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())), m_Value(), m_Value()))) return InstDesc(false, I); // Look for a min/max pattern. if (match(I, m_UMin(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::UMin); + return InstDesc(Kind == RecurKind::UMin, I); if (match(I, m_UMax(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::UMax); + return InstDesc(Kind == RecurKind::UMax, I); if (match(I, m_SMax(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::SMax); + return InstDesc(Kind == RecurKind::SMax, I); if (match(I, m_SMin(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::SMin); + return InstDesc(Kind == RecurKind::SMin, I); if (match(I, m_OrdFMin(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::FMin); + return InstDesc(Kind == RecurKind::FMin, I); if (match(I, m_OrdFMax(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::FMax); + return InstDesc(Kind == RecurKind::FMax, I); if (match(I, m_UnordFMin(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::FMin); + return InstDesc(Kind == RecurKind::FMin, I); if (match(I, m_UnordFMax(m_Value(), m_Value()))) - return InstDesc(I, RecurKind::FMax); + return InstDesc(Kind == RecurKind::FMax, I); + if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) + return InstDesc(Kind == RecurKind::FMin, I); + if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) + return InstDesc(Kind == RecurKind::FMax, I); return InstDesc(false, I); } @@ -592,8 +667,10 @@ RecurrenceDescriptor::isConditionalRdxPattern(RecurKind Kind, Instruction *I) { } RecurrenceDescriptor::InstDesc -RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind, - InstDesc &Prev, FastMathFlags FMF) { +RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi, + Instruction *I, RecurKind Kind, + InstDesc &Prev, FastMathFlags FuncFMF) { + assert(Prev.getRecKind() == RecurKind::None || Prev.getRecKind() == Kind); switch (I->getOpcode()) { default: return InstDesc(false, I); @@ -624,9 +701,15 @@ RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind, LLVM_FALLTHROUGH; case Instruction::FCmp: case Instruction::ICmp: + case Instruction::Call: + if (isSelectCmpRecurrenceKind(Kind)) + return isSelectCmpPattern(L, OrigPhi, I, Prev); if (isIntMinMaxRecurrenceKind(Kind) || - (FMF.noNaNs() && FMF.noSignedZeros() && isFPMinMaxRecurrenceKind(Kind))) - return isMinMaxSelectCmpPattern(I, Prev); + (((FuncFMF.noNaNs() && FuncFMF.noSignedZeros()) || + (isa<FPMathOperator>(I) && I->hasNoNaNs() && + I->hasNoSignedZeros())) && + isFPMinMaxRecurrenceKind(Kind))) + return isMinMaxPattern(I, Kind, Prev); return InstDesc(false, I); } } @@ -649,7 +732,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes, DemandedBits *DB, AssumptionCache *AC, DominatorTree *DT) { - BasicBlock *Header = TheLoop->getHeader(); Function &F = *Header->getParent(); FastMathFlags FMF; @@ -694,6 +776,12 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n"); return true; } + if (AddReductionVar(Phi, RecurKind::SelectICmp, TheLoop, FMF, RedDes, DB, AC, + DT)) { + LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI." + << *Phi << "\n"); + return true; + } if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n"); return true; @@ -710,6 +798,12 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n"); return true; } + if (AddReductionVar(Phi, RecurKind::SelectFCmp, TheLoop, FMF, RedDes, DB, AC, + DT)) { + LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI." + << " PHI." << *Phi << "\n"); + return true; + } // Not a reduction of known type. return false; } @@ -816,8 +910,8 @@ bool RecurrenceDescriptor::isFirstOrderRecurrence( /// This function returns the identity element (or neutral element) for /// the operation K. -Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurKind K, Type *Tp, - FastMathFlags FMF) { +Value *RecurrenceDescriptor::getRecurrenceIdentity(RecurKind K, Type *Tp, + FastMathFlags FMF) { switch (K) { case RecurKind::Xor: case RecurKind::Add: @@ -857,6 +951,10 @@ Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurKind K, Type *Tp, return ConstantFP::getInfinity(Tp, true); case RecurKind::FMax: return ConstantFP::getInfinity(Tp, false); + case RecurKind::SelectICmp: + case RecurKind::SelectFCmp: + return getRecurrenceStartValue(); + break; default: llvm_unreachable("Unknown recurrence kind"); } @@ -882,9 +980,11 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) { case RecurKind::SMin: case RecurKind::UMax: case RecurKind::UMin: + case RecurKind::SelectICmp: return Instruction::ICmp; case RecurKind::FMax: case RecurKind::FMin: + case RecurKind::SelectFCmp: return Instruction::FCmp; default: llvm_unreachable("Unknown recurrence operation"); @@ -963,8 +1063,10 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const { InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, BinaryOperator *BOp, + Type *ElementType, SmallVectorImpl<Instruction *> *Casts) - : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp) { + : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp), + ElementType(ElementType) { assert(IK != IK_NoInduction && "Not an induction"); // Start value type should match the induction kind and the value @@ -992,6 +1094,11 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, InductionBinOp->getOpcode() == Instruction::FSub))) && "Binary opcode should be specified for FP induction"); + if (IK == IK_PtrInduction) + assert(ElementType && "Pointer induction must have element type"); + else + assert(!ElementType && "Non-pointer induction cannot have element type"); + if (Casts) { for (auto &Inst : *Casts) { RedundantCasts.push_back(Inst); @@ -1239,8 +1346,6 @@ bool InductionDescriptor::isInductionPHI( BasicBlock *Latch = AR->getLoop()->getLoopLatch(); if (!Latch) return false; - BinaryOperator *BOp = - dyn_cast<BinaryOperator>(Phi->getIncomingValueForBlock(Latch)); const SCEV *Step = AR->getStepRecurrence(*SE); // Calculate the pointer stride and check if it is consecutive. @@ -1250,8 +1355,10 @@ bool InductionDescriptor::isInductionPHI( return false; if (PhiTy->isIntegerTy()) { + BinaryOperator *BOp = + dyn_cast<BinaryOperator>(Phi->getIncomingValueForBlock(Latch)); D = InductionDescriptor(StartValue, IK_IntInduction, Step, BOp, - CastsToIgnore); + /* ElementType */ nullptr, CastsToIgnore); return true; } @@ -1260,15 +1367,16 @@ bool InductionDescriptor::isInductionPHI( if (!ConstStep) return false; - ConstantInt *CV = ConstStep->getValue(); - Type *PointerElementType = PhiTy->getPointerElementType(); - // The pointer stride cannot be determined if the pointer element type is not - // sized. - if (!PointerElementType->isSized()) + // Always use i8 element type for opaque pointer inductions. + PointerType *PtrTy = cast<PointerType>(PhiTy); + Type *ElementType = PtrTy->isOpaque() ? Type::getInt8Ty(PtrTy->getContext()) + : PtrTy->getElementType(); + if (!ElementType->isSized()) return false; + ConstantInt *CV = ConstStep->getValue(); const DataLayout &DL = Phi->getModule()->getDataLayout(); - int64_t Size = static_cast<int64_t>(DL.getTypeAllocSize(PointerElementType)); + int64_t Size = static_cast<int64_t>(DL.getTypeAllocSize(ElementType)); if (!Size) return false; @@ -1277,6 +1385,7 @@ bool InductionDescriptor::isInductionPHI( return false; auto *StepValue = SE->getConstant(CV->getType(), CVSize / Size, true /* signed */); - D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, BOp); + D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, + /* BinOp */ nullptr, ElementType); return true; } diff --git a/llvm/lib/Analysis/IVUsers.cpp b/llvm/lib/Analysis/IVUsers.cpp index db6cff720642..d7b202f83189 100644 --- a/llvm/lib/Analysis/IVUsers.cpp +++ b/llvm/lib/Analysis/IVUsers.cpp @@ -90,34 +90,6 @@ static bool isInteresting(const SCEV *S, const Instruction *I, const Loop *L, return false; } -/// Return true if all loop headers that dominate this block are in simplified -/// form. -static bool isSimplifiedLoopNest(BasicBlock *BB, const DominatorTree *DT, - const LoopInfo *LI, - SmallPtrSetImpl<Loop*> &SimpleLoopNests) { - Loop *NearestLoop = nullptr; - for (DomTreeNode *Rung = DT->getNode(BB); - Rung; Rung = Rung->getIDom()) { - BasicBlock *DomBB = Rung->getBlock(); - Loop *DomLoop = LI->getLoopFor(DomBB); - if (DomLoop && DomLoop->getHeader() == DomBB) { - // If we have already checked this loop nest, stop checking. - if (SimpleLoopNests.count(DomLoop)) - break; - // If the domtree walk reaches a loop with no preheader, return false. - if (!DomLoop->isLoopSimplifyForm()) - return false; - // If we have not already checked this loop nest, remember the loop - // header nearest to BB. The nearest loop may not contain BB. - if (!NearestLoop) - NearestLoop = DomLoop; - } - } - if (NearestLoop) - SimpleLoopNests.insert(NearestLoop); - return true; -} - /// IVUseShouldUsePostIncValue - We have discovered a "User" of an IV expression /// and now we need to decide whether the user should use the preinc or post-inc /// value. If this user should use the post-inc version of the IV, return true. @@ -162,11 +134,10 @@ static bool IVUseShouldUsePostIncValue(Instruction *User, Value *Operand, return true; } -/// AddUsersImpl - Inspect the specified instruction. If it is a -/// reducible SCEV, recursively add its users to the IVUsesByStride set and -/// return true. Otherwise, return false. -bool IVUsers::AddUsersImpl(Instruction *I, - SmallPtrSetImpl<Loop*> &SimpleLoopNests) { +/// Inspect the specified instruction. If it is a reducible SCEV, recursively +/// add its users to the IVUsesByStride set and return true. Otherwise, return +/// false. +bool IVUsers::AddUsersIfInteresting(Instruction *I) { const DataLayout &DL = I->getModule()->getDataLayout(); // Add this IV user to the Processed set before returning false to ensure that @@ -213,18 +184,6 @@ bool IVUsers::AddUsersImpl(Instruction *I, if (isa<PHINode>(User) && Processed.count(User)) continue; - // Only consider IVUsers that are dominated by simplified loop - // headers. Otherwise, SCEVExpander will crash. - BasicBlock *UseBB = User->getParent(); - // A phi's use is live out of its predecessor block. - if (PHINode *PHI = dyn_cast<PHINode>(User)) { - unsigned OperandNo = U.getOperandNo(); - unsigned ValNo = PHINode::getIncomingValueNumForOperand(OperandNo); - UseBB = PHI->getIncomingBlock(ValNo); - } - if (!isSimplifiedLoopNest(UseBB, DT, LI, SimpleLoopNests)) - return false; - // Descend recursively, but not into PHI nodes outside the current loop. // It's important to see the entire expression outside the loop to get // choices that depend on addressing mode use right, although we won't @@ -234,12 +193,12 @@ bool IVUsers::AddUsersImpl(Instruction *I, bool AddUserToIVUsers = false; if (LI->getLoopFor(User->getParent()) != L) { if (isa<PHINode>(User) || Processed.count(User) || - !AddUsersImpl(User, SimpleLoopNests)) { + !AddUsersIfInteresting(User)) { LLVM_DEBUG(dbgs() << "FOUND USER in other loop: " << *User << '\n' << " OF SCEV: " << *ISE << '\n'); AddUserToIVUsers = true; } - } else if (Processed.count(User) || !AddUsersImpl(User, SimpleLoopNests)) { + } else if (Processed.count(User) || !AddUsersIfInteresting(User)) { LLVM_DEBUG(dbgs() << "FOUND USER: " << *User << '\n' << " OF SCEV: " << *ISE << '\n'); AddUserToIVUsers = true; @@ -288,15 +247,6 @@ bool IVUsers::AddUsersImpl(Instruction *I, return true; } -bool IVUsers::AddUsersIfInteresting(Instruction *I) { - // SCEVExpander can only handle users that are dominated by simplified loop - // entries. Keep track of all loops that are only dominated by other simple - // loops so we don't traverse the domtree for each user. - SmallPtrSet<Loop*,16> SimpleLoopNests; - - return AddUsersImpl(I, SimpleLoopNests); -} - IVStrideUse &IVUsers::AddUser(Instruction *User, Value *Operand) { IVUses.push_back(new IVStrideUse(this, User, Operand)); return IVUses.back(); diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp index a8ad2d6696bf..73d1eff1b968 100644 --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -49,6 +49,42 @@ static cl::opt<int> extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats; +namespace { +using namespace llvm::ore; +class MandatoryInlineAdvice : public InlineAdvice { +public: + MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, + OptimizationRemarkEmitter &ORE, + bool IsInliningMandatory) + : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {} + +private: + void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); } + + void recordInliningImpl() override { + if (IsInliningRecommended) + emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended, + [&](OptimizationRemark &Remark) { + Remark << ": always inline attribute"; + }); + } + + void recordUnsuccessfulInliningImpl(const InlineResult &Result) override { + if (IsInliningRecommended) + ORE.emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) + << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '" + << NV("Caller", Caller) + << "': " << NV("Reason", Result.getFailureReason()); + }); + } + + void recordUnattemptedInliningImpl() override { + assert(!IsInliningRecommended && "Expected to attempt inlining"); + } +}; +} // namespace + void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( const InlineResult &Result) { using namespace ore; @@ -56,20 +92,20 @@ void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( "; " + inlineCostStr(*OIC)); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) - << NV("Callee", Callee) << " will not be inlined into " - << NV("Caller", Caller) << ": " - << NV("Reason", Result.getFailureReason()); + << "'" << NV("Callee", Callee) << "' is not inlined into '" + << NV("Caller", Caller) + << "': " << NV("Reason", Result.getFailureReason()); }); } void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() { if (EmitRemarks) - emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC); + emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC); } void DefaultInlineAdvice::recordInliningImpl() { if (EmitRemarks) - emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC); + emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC); } llvm::Optional<llvm::InlineCost> static getDefaultInlineAdvice( @@ -151,9 +187,9 @@ void InlineAdvice::recordInliningWithCalleeDeleted() { AnalysisKey InlineAdvisorAnalysis::Key; -bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params, - InliningAdvisorMode Mode, - StringRef ReplayFile) { +bool InlineAdvisorAnalysis::Result::tryCreate( + InlineParams Params, InliningAdvisorMode Mode, + const ReplayInlinerSettings &ReplaySettings) { auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); switch (Mode) { case InliningAdvisorMode::Default: @@ -161,10 +197,10 @@ bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params, Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params)); // Restrict replay to default advisor, ML advisors are stateful so // replay will need augmentations to interleave with them correctly. - if (!ReplayFile.empty()) { - Advisor = std::make_unique<ReplayInlineAdvisor>( - M, FAM, M.getContext(), std::move(Advisor), ReplayFile, - /* EmitRemarks =*/true); + if (!ReplaySettings.ReplayFile.empty()) { + Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(), + std::move(Advisor), ReplaySettings, + /* EmitRemarks =*/true); } break; case InliningAdvisorMode::Development: @@ -313,7 +349,7 @@ void llvm::setInlineRemark(CallBase &CB, StringRef Message) { return; Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message); - CB.addAttribute(AttributeList::FunctionIndex, Attr); + CB.addFnAttr(Attr); } /// Return the cost only if the inliner should attempt to inline at the given @@ -343,15 +379,15 @@ llvm::shouldInline(CallBase &CB, if (IC.isNever()) { ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call) - << NV("Callee", Callee) << " not inlined into " - << NV("Caller", Caller) << " because it should never be inlined " - << IC; + << "'" << NV("Callee", Callee) << "' not inlined into '" + << NV("Caller", Caller) + << "' because it should never be inlined " << IC; }); } else { ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call) - << NV("Callee", Callee) << " not inlined into " - << NV("Caller", Caller) << " because too costly to inline " + << "'" << NV("Callee", Callee) << "' not inlined into '" + << NV("Caller", Caller) << "' because too costly to inline " << IC; }); } @@ -368,9 +404,9 @@ llvm::shouldInline(CallBase &CB, ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts", Call) - << "Not inlining. Cost of inlining " << NV("Callee", Callee) - << " increases the cost of inlining " << NV("Caller", Caller) - << " in other contexts"; + << "Not inlining. Cost of inlining '" << NV("Callee", Callee) + << "' increases the cost of inlining '" << NV("Caller", Caller) + << "' in other contexts"; }); setInlineRemark(CB, "deferred"); // IC does not bool() to false, so get an InlineCost that will. @@ -383,7 +419,8 @@ llvm::shouldInline(CallBase &CB, return IC; } -std::string llvm::getCallSiteLocation(DebugLoc DLoc) { +std::string llvm::formatCallSiteLocation(DebugLoc DLoc, + const CallSiteFormat &Format) { std::string Buffer; raw_string_ostream CallSiteLoc(Buffer); bool First = true; @@ -399,9 +436,10 @@ std::string llvm::getCallSiteLocation(DebugLoc DLoc) { StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); if (Name.empty()) Name = DIL->getScope()->getSubprogram()->getName(); - CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset) << ":" - << llvm::utostr(DIL->getColumn()); - if (Discriminator) + CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset); + if (Format.outputColumn()) + CallSiteLoc << ":" << llvm::utostr(DIL->getColumn()); + if (Format.outputDiscriminator() && Discriminator) CallSiteLoc << "." << llvm::utostr(Discriminator); First = false; } @@ -435,25 +473,38 @@ void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) { Remark << ";"; } -void llvm::emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, - const BasicBlock *Block, const Function &Callee, - const Function &Caller, const InlineCost &IC, - bool ForProfileContext, const char *PassName) { +void llvm::emitInlinedInto( + OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, + const Function &Callee, const Function &Caller, bool AlwaysInline, + function_ref<void(OptimizationRemark &)> ExtraContext, + const char *PassName) { ORE.emit([&]() { - bool AlwaysInline = IC.isAlways(); StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined"; OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName, DLoc, Block); - Remark << ore::NV("Callee", &Callee) << " inlined into "; - Remark << ore::NV("Caller", &Caller); - if (ForProfileContext) - Remark << " to match profiling context"; - Remark << " with " << IC; + Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '" + << ore::NV("Caller", &Caller) << "'"; + if (ExtraContext) + ExtraContext(Remark); addLocationToRemarks(Remark, DLoc); return Remark; }); } +void llvm::emitInlinedIntoBasedOnCost( + OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, + const Function &Callee, const Function &Caller, const InlineCost &IC, + bool ForProfileContext, const char *PassName) { + llvm::emitInlinedInto( + ORE, DLoc, Block, Callee, Caller, IC.isAlways(), + [&](OptimizationRemark &Remark) { + if (ForProfileContext) + Remark << " to match profiling context"; + Remark << " with " << IC; + }, + PassName); +} + InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM) : M(M), FAM(FAM) { if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { @@ -475,7 +526,8 @@ InlineAdvisor::~InlineAdvisor() { std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB, bool Advice) { - return std::make_unique<InlineAdvice>(this, CB, getCallerORE(CB), Advice); + return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB), + Advice); } InlineAdvisor::MandatoryInliningKind diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp index 4c2413e14435..ff31e81aad08 100644 --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -135,6 +135,31 @@ static cl::opt<bool> DisableGEPConstOperand( namespace { class InlineCostCallAnalyzer; +/// This function behaves more like CallBase::hasFnAttr: when it looks for the +/// requested attribute, it check both the call instruction and the called +/// function (if it's available and operand bundles don't prohibit that). +Attribute getFnAttr(CallBase &CB, StringRef AttrKind) { + Attribute CallAttr = CB.getFnAttr(AttrKind); + if (CallAttr.isValid()) + return CallAttr; + + // Operand bundles override attributes on the called function, but don't + // override attributes directly present on the call instruction. + if (!CB.isFnAttrDisallowedByOpBundle(AttrKind)) + if (const Function *F = CB.getCalledFunction()) + return F->getFnAttribute(AttrKind); + + return {}; +} + +Optional<int> getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind) { + Attribute Attr = getFnAttr(CB, AttrKind); + int AttrValue; + if (Attr.getValueAsString().getAsInteger(10, AttrValue)) + return None; + return AttrValue; +} + // This struct is used to store information about inline cost of a // particular instruction struct InstructionCostDetail { @@ -235,6 +260,10 @@ protected: /// Called the analysis engine determines load elimination won't happen. virtual void onDisableLoadElimination() {} + /// Called when we visit a CallBase, before the analysis starts. Return false + /// to stop further processing of the instruction. + virtual bool onCallBaseVisitStart(CallBase &Call) { return true; } + /// Called to account for a call. virtual void onCallPenalty() {} @@ -333,6 +362,10 @@ protected: /// whenever we simplify away the stores that would otherwise cause them to be /// loads. bool EnableLoadElimination; + + /// Whether we allow inlining for recursive call. + bool AllowRecursiveCall; + SmallPtrSet<Value *, 16> LoadAddrSet; AllocaInst *getSROAArgForValueOrNull(Value *V) const { @@ -354,6 +387,7 @@ protected: bool simplifyCallSite(Function *F, CallBase &Call); template <typename Callable> bool simplifyInstruction(Instruction &I, Callable Evaluate); + bool simplifyIntrinsicCallIsConstant(CallBase &CB); ConstantInt *stripAndComputeInBoundsConstantOffsets(Value *&V); /// Return true if the given argument to the function being considered for @@ -421,7 +455,8 @@ public: OptimizationRemarkEmitter *ORE = nullptr) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), GetBFI(GetBFI), PSI(PSI), F(Callee), DL(F.getParent()->getDataLayout()), ORE(ORE), - CandidateCall(Call), EnableLoadElimination(true) {} + CandidateCall(Call), EnableLoadElimination(true), + AllowRecursiveCall(false) {} InlineResult analyze(); @@ -510,6 +545,9 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // sense that it's not weighted by profile counts at all. int ColdSize = 0; + // Whether inlining is decided by cost-threshold analysis. + bool DecidedByCostThreshold = false; + // Whether inlining is decided by cost-benefit analysis. bool DecidedByCostBenefit = false; @@ -558,6 +596,22 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { addCost(LoadEliminationCost); LoadEliminationCost = 0; } + + bool onCallBaseVisitStart(CallBase &Call) override { + if (Optional<int> AttrCallThresholdBonus = + getStringFnAttrAsInt(Call, "call-threshold-bonus")) + Threshold += *AttrCallThresholdBonus; + + if (Optional<int> AttrCallCost = + getStringFnAttrAsInt(Call, "call-inline-cost")) { + addCost(*AttrCallCost); + // Prevent further processing of the call since we want to override its + // inline cost, not just add to it. + return false; + } + return true; + } + void onCallPenalty() override { addCost(CallPenalty); } void onCallArgumentSetup(const CallBase &Call) override { // Pay the price of the argument setup. We account for the average 1 @@ -717,7 +771,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // Make sure we have a nonzero entry count. auto EntryCount = F.getEntryCount(); - if (!EntryCount || !EntryCount.getCount()) + if (!EntryCount || !EntryCount->getCount()) return false; BlockFrequencyInfo *CalleeBFI = &(GetBFI(F)); @@ -763,7 +817,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { if (BranchInst *BI = dyn_cast<BranchInst>(&I)) { // Count a conditional branch as savings if it becomes unconditional. if (BI->isConditional() && - dyn_cast_or_null<ConstantInt>( + isa_and_nonnull<ConstantInt>( SimplifiedValues.lookup(BI->getCondition()))) { CurrentSavings += InlineConstants::InstrCost; } @@ -783,8 +837,8 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // Compute the cycle savings per call. auto EntryProfileCount = F.getEntryCount(); - assert(EntryProfileCount.hasValue() && EntryProfileCount.getCount()); - auto EntryCount = EntryProfileCount.getCount(); + assert(EntryProfileCount.hasValue() && EntryProfileCount->getCount()); + auto EntryCount = EntryProfileCount->getCount(); CycleSavings += EntryCount / 2; CycleSavings = CycleSavings.udiv(EntryCount); @@ -847,6 +901,14 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { else if (NumVectorInstructions <= NumInstructions / 2) Threshold -= VectorBonus / 2; + if (Optional<int> AttrCost = + getStringFnAttrAsInt(CandidateCall, "function-inline-cost")) + Cost = *AttrCost; + + if (Optional<int> AttrThreshold = + getStringFnAttrAsInt(CandidateCall, "function-inline-threshold")) + Threshold = *AttrThreshold; + if (auto Result = costBenefitAnalysis()) { DecidedByCostBenefit = true; if (Result.getValue()) @@ -855,14 +917,24 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { return InlineResult::failure("Cost over threshold."); } - if (IgnoreThreshold || Cost < std::max(1, Threshold)) + if (IgnoreThreshold) return InlineResult::success(); - return InlineResult::failure("Cost over threshold."); + + DecidedByCostThreshold = true; + return Cost < std::max(1, Threshold) + ? InlineResult::success() + : InlineResult::failure("Cost over threshold."); } + bool shouldStop() override { + if (IgnoreThreshold || ComputeFullInlineCost) + return false; // Bail out the moment we cross the threshold. This means we'll under-count // the cost, but only when undercounting doesn't matter. - return !IgnoreThreshold && Cost >= Threshold && !ComputeFullInlineCost; + if (Cost < Threshold) + return false; + DecidedByCostThreshold = true; + return true; } void onLoadEliminationOpportunity() override { @@ -930,7 +1002,9 @@ public: Params(Params), Threshold(Params.DefaultThreshold), BoostIndirectCalls(BoostIndirect), IgnoreThreshold(IgnoreThreshold), CostBenefitAnalysisEnabled(isCostBenefitAnalysisEnabled()), - Writer(this) {} + Writer(this) { + AllowRecursiveCall = Params.AllowRecursiveCall.getValue(); + } /// Annotation Writer for instruction details InlineCostAnnotationWriter Writer; @@ -939,7 +1013,7 @@ public: // Prints the same analysis as dump(), but its definition is not dependent // on the build. - void print(); + void print(raw_ostream &OS); Optional<InstructionCostDetail> getCostDetails(const Instruction *I) { if (InstructionCostDetailMap.find(I) != InstructionCostDetailMap.end()) @@ -952,6 +1026,7 @@ public: int getCost() const { return Cost; } Optional<CostBenefitPair> getCostBenefitPair() { return CostBenefit; } bool wasDecidedByCostBenefit() const { return DecidedByCostBenefit; } + bool wasDecidedByCostThreshold() const { return DecidedByCostThreshold; } }; class InlineCostFeaturesAnalyzer final : public CallAnalyzer { @@ -1310,7 +1385,7 @@ bool CallAnalyzer::visitPHI(PHINode &I) { // Or could we skip the getPointerSizeInBits call completely? As far as I can // see the ZeroOffset is used as a dummy value, so we can probably use any // bit width for the ZeroOffset? - APInt ZeroOffset = APInt::getNullValue(DL.getPointerSizeInBits(0)); + APInt ZeroOffset = APInt::getZero(DL.getPointerSizeInBits(0)); bool CheckSROA = I.getType()->isPointerTy(); // Track the constant or pointer with constant offset we've seen so far. @@ -1471,6 +1546,27 @@ bool CallAnalyzer::simplifyInstruction(Instruction &I, Callable Evaluate) { return true; } +/// Try to simplify a call to llvm.is.constant. +/// +/// Duplicate the argument checking from CallAnalyzer::simplifyCallSite since +/// we expect calls of this specific intrinsic to be infrequent. +/// +/// FIXME: Given that we know CB's parent (F) caller +/// (CandidateCall->getParent()->getParent()), we might be able to determine +/// whether inlining F into F's caller would change how the call to +/// llvm.is.constant would evaluate. +bool CallAnalyzer::simplifyIntrinsicCallIsConstant(CallBase &CB) { + Value *Arg = CB.getArgOperand(0); + auto *C = dyn_cast<Constant>(Arg); + + if (!C) + C = dyn_cast_or_null<Constant>(SimplifiedValues.lookup(Arg)); + + Type *RT = CB.getFunctionType()->getReturnType(); + SimplifiedValues[&CB] = ConstantInt::get(RT, C ? 1 : 0); + return true; +} + bool CallAnalyzer::visitBitCast(BitCastInst &I) { // Propagate constants through bitcasts. if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { @@ -1799,8 +1895,8 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { SingleBBBonus = Threshold * SingleBBBonusPercent / 100; VectorBonus = Threshold * VectorBonusPercent / 100; - bool OnlyOneCallAndLocalLinkage = - F.hasLocalLinkage() && F.hasOneUse() && &F == Call.getCalledFunction(); + bool OnlyOneCallAndLocalLinkage = F.hasLocalLinkage() && F.hasOneLiveUse() && + &F == Call.getCalledFunction(); // If there is only one call of the function, and it has internal linkage, // the cost of inlining it drops dramatically. It may seem odd to update // Cost in updateThreshold, but the bonus depends on the logic in this method. @@ -2029,6 +2125,9 @@ bool CallAnalyzer::simplifyCallSite(Function *F, CallBase &Call) { } bool CallAnalyzer::visitCallBase(CallBase &Call) { + if (!onCallBaseVisitStart(Call)) + return true; + if (Call.hasFnAttr(Attribute::ReturnsTwice) && !F.hasFnAttribute(Attribute::ReturnsTwice)) { // This aborts the entire analysis. @@ -2091,6 +2190,8 @@ bool CallAnalyzer::visitCallBase(CallBase &Call) { if (auto *SROAArg = getSROAArgForValueOrNull(II->getOperand(0))) SROAArgValues[II] = SROAArg; return true; + case Intrinsic::is_constant: + return simplifyIntrinsicCallIsConstant(Call); } } @@ -2098,7 +2199,8 @@ bool CallAnalyzer::visitCallBase(CallBase &Call) { // This flag will fully abort the analysis, so don't bother with anything // else. IsRecursiveCall = true; - return false; + if (!AllowRecursiveCall) + return false; } if (TTI.isLoweredToCall(F)) { @@ -2123,7 +2225,7 @@ bool CallAnalyzer::visitBranchInst(BranchInst &BI) { // inliner more regular and predictable. Interestingly, conditional branches // which will fold away are also free. return BI.isUnconditional() || isa<ConstantInt>(BI.getCondition()) || - dyn_cast_or_null<ConstantInt>( + isa_and_nonnull<ConstantInt>( SimplifiedValues.lookup(BI.getCondition())); } @@ -2305,11 +2407,8 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB, // inlining due to debug symbols. Eventually, the number of unsimplified // instructions shouldn't factor into the cost computation, but until then, // hack around it here. - if (isa<DbgInfoIntrinsic>(I)) - continue; - - // Skip pseudo-probes. - if (isa<PseudoProbeInst>(I)) + // Similarly, skip pseudo-probes. + if (I.isDebugOrPseudoInst()) continue; // Skip ephemeral values. @@ -2336,7 +2435,7 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB, using namespace ore; // If the visit this instruction detected an uninlinable pattern, abort. InlineResult IR = InlineResult::success(); - if (IsRecursiveCall) + if (IsRecursiveCall && !AllowRecursiveCall) IR = InlineResult::failure("recursive"); else if (ExposesReturnsTwice) IR = InlineResult::failure("exposes returns twice"); @@ -2398,7 +2497,7 @@ ConstantInt *CallAnalyzer::stripAndComputeInBoundsConstantOffsets(Value *&V) { unsigned AS = V->getType()->getPointerAddressSpace(); unsigned IntPtrWidth = DL.getIndexSizeInBits(AS); - APInt Offset = APInt::getNullValue(IntPtrWidth); + APInt Offset = APInt::getZero(IntPtrWidth); // Even though we don't look through PHI nodes, we could be called on an // instruction in an unreachable block, which may be on a cycle. @@ -2601,7 +2700,7 @@ InlineResult CallAnalyzer::analyze() { onBlockAnalyzed(BB); } - bool OnlyOneCallAndLocalLinkage = F.hasLocalLinkage() && F.hasOneUse() && + bool OnlyOneCallAndLocalLinkage = F.hasLocalLinkage() && F.hasOneLiveUse() && &F == CandidateCall.getCalledFunction(); // If this is a noduplicate call, we can still inline as long as // inlining this would cause the removal of the caller (so the instruction @@ -2612,10 +2711,10 @@ InlineResult CallAnalyzer::analyze() { return finalizeAnalysis(); } -void InlineCostCallAnalyzer::print() { -#define DEBUG_PRINT_STAT(x) dbgs() << " " #x ": " << x << "\n" +void InlineCostCallAnalyzer::print(raw_ostream &OS) { +#define DEBUG_PRINT_STAT(x) OS << " " #x ": " << x << "\n" if (PrintInstructionComments) - F.print(dbgs(), &Writer); + F.print(OS, &Writer); DEBUG_PRINT_STAT(NumConstantArgs); DEBUG_PRINT_STAT(NumConstantOffsetPtrArgs); DEBUG_PRINT_STAT(NumAllocaArgs); @@ -2634,7 +2733,7 @@ void InlineCostCallAnalyzer::print() { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Dump stats about this call's analysis. -LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() { print(); } +LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() { print(dbgs()); } #endif /// Test that there are no attribute conflicts between Caller and Callee @@ -2849,13 +2948,13 @@ InlineCost llvm::getInlineCost( return InlineCost::getNever("cost over benefit", CA.getCostBenefitPair()); } - // Check if there was a reason to force inlining or no inlining. - if (!ShouldInline.isSuccess() && CA.getCost() < CA.getThreshold()) - return InlineCost::getNever(ShouldInline.getFailureReason()); - if (ShouldInline.isSuccess() && CA.getCost() >= CA.getThreshold()) - return InlineCost::getAlways("empty function"); + if (CA.wasDecidedByCostThreshold()) + return InlineCost::get(CA.getCost(), CA.getThreshold()); - return llvm::InlineCost::get(CA.getCost(), CA.getThreshold()); + // No details on how the decision was made, simply return always or never. + return ShouldInline.isSuccess() + ? InlineCost::getAlways("empty function") + : InlineCost::getNever(ShouldInline.getFailureReason()); } InlineResult llvm::isInlineViable(Function &F) { @@ -3028,7 +3127,8 @@ InlineCostAnnotationPrinterPass::run(Function &F, ICCA.analyze(); OS << " Analyzing call of " << CalledFunction->getName() << "... (caller:" << CI->getCaller()->getName() << ")\n"; - ICCA.print(); + ICCA.print(OS); + OS << "\n"; } } } diff --git a/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp b/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp index 3c90e82fb952..a2e231e2d0f4 100644 --- a/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp +++ b/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp @@ -1,9 +1,8 @@ //===- InlineSizeEstimatorAnalysis.cpp - IR to native size from ML model --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp b/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp index 7d1e630e6e80..9fee57c54b85 100644 --- a/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp +++ b/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp @@ -19,11 +19,15 @@ #include "llvm/Analysis/InstructionPrecedenceTracking.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/ADT/Statistic.h" #include "llvm/IR/PatternMatch.h" #include "llvm/Support/CommandLine.h" using namespace llvm; +#define DEBUG_TYPE "ipt" +STATISTIC(NumInstScanned, "Number of insts scanned while updating ibt"); + #ifndef NDEBUG static cl::opt<bool> ExpensiveAsserts( "ipt-expensive-asserts", @@ -64,11 +68,13 @@ bool InstructionPrecedenceTracking::isPreceededBySpecialInstruction( void InstructionPrecedenceTracking::fill(const BasicBlock *BB) { FirstSpecialInsts.erase(BB); - for (auto &I : *BB) + for (auto &I : *BB) { + NumInstScanned++; if (isSpecialInstruction(&I)) { FirstSpecialInsts[BB] = &I; return; } + } // Mark this block as having no special instructions. FirstSpecialInsts[BB] = nullptr; @@ -107,8 +113,10 @@ void InstructionPrecedenceTracking::insertInstructionTo(const Instruction *Inst, } void InstructionPrecedenceTracking::removeInstruction(const Instruction *Inst) { - if (isSpecialInstruction(Inst)) - FirstSpecialInsts.erase(Inst->getParent()); + auto *BB = Inst->getParent(); + assert(BB && "must be called before instruction is actually removed"); + if (FirstSpecialInsts.count(BB) && FirstSpecialInsts[BB] == Inst) + FirstSpecialInsts.erase(BB); } void InstructionPrecedenceTracking::removeUsersOf(const Instruction *Inst) { diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 23083bc8178e..864eeea4f8bf 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -70,8 +70,8 @@ static Value *SimplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned); static Value *SimplifyXorInst(Value *, Value *, const SimplifyQuery &, unsigned); static Value *SimplifyCastInst(unsigned, Value *, Type *, const SimplifyQuery &, unsigned); -static Value *SimplifyGEPInst(Type *, ArrayRef<Value *>, const SimplifyQuery &, - unsigned); +static Value *SimplifyGEPInst(Type *, ArrayRef<Value *>, bool, + const SimplifyQuery &, unsigned); static Value *SimplifySelectInst(Value *, Value *, Value *, const SimplifyQuery &, unsigned); @@ -698,13 +698,12 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, bool AllowNonInbounds = false) { assert(V->getType()->isPtrOrPtrVectorTy()); - Type *IntIdxTy = DL.getIndexType(V->getType())->getScalarType(); - APInt Offset = APInt::getNullValue(IntIdxTy->getIntegerBitWidth()); + APInt Offset = APInt::getZero(DL.getIndexTypeSizeInBits(V->getType())); V = V->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds); // As that strip may trace through `addrspacecast`, need to sext or trunc // the offset calculated. - IntIdxTy = DL.getIndexType(V->getType())->getScalarType(); + Type *IntIdxTy = DL.getIndexType(V->getType())->getScalarType(); Offset = Offset.sextOrTrunc(IntIdxTy->getIntegerBitWidth()); Constant *OffsetIntPtr = ConstantInt::get(IntIdxTy, Offset); @@ -1407,8 +1406,7 @@ static Value *SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, match(Op0, m_c_Or(m_NUWShl(m_Value(X), m_APInt(ShLAmt)), m_Value(Y))) && *ShRAmt == *ShLAmt) { const KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); - const unsigned Width = Op0->getType()->getScalarSizeInBits(); - const unsigned EffWidthY = Width - YKnown.countMinLeadingZeros(); + const unsigned EffWidthY = YKnown.countMaxActiveBits(); if (ShRAmt->uge(EffWidthY)) return X; } @@ -1429,9 +1427,11 @@ static Value *SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, MaxRecurse)) return V; - // all ones >>a X -> -1 + // -1 >>a X --> -1 + // (-1 << X) a>> X --> -1 // Do not return Op0 because it may contain undef elements if it's a vector. - if (match(Op0, m_AllOnes())) + if (match(Op0, m_AllOnes()) || + match(Op0, m_Shl(m_AllOnes(), m_Specific(Op1)))) return Constant::getAllOnesValue(Op0->getType()); // (X << A) >> A -> X @@ -1765,7 +1765,7 @@ static Value *simplifyAndOrOfICmpsWithLimitConst(ICmpInst *Cmp0, ICmpInst *Cmp1, if (match(Cmp0->getOperand(1), m_APInt(C))) MinMaxC = HasNotOp ? ~*C : *C; else if (isa<ConstantPointerNull>(Cmp0->getOperand(1))) - MinMaxC = APInt::getNullValue(8); + MinMaxC = APInt::getZero(8); else return nullptr; @@ -2040,24 +2040,32 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, if (match(Op1, m_c_Or(m_Specific(Op0), m_Value()))) return Op0; + // (X | Y) & (X | ~Y) --> X (commuted 8 ways) + Value *X, *Y; + if (match(Op0, m_c_Or(m_Value(X), m_Not(m_Value(Y)))) && + match(Op1, m_c_Or(m_Deferred(X), m_Deferred(Y)))) + return X; + if (match(Op1, m_c_Or(m_Value(X), m_Not(m_Value(Y)))) && + match(Op0, m_c_Or(m_Deferred(X), m_Deferred(Y)))) + return X; + if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::And)) return V; // A mask that only clears known zeros of a shifted value is a no-op. - Value *X; const APInt *Mask; const APInt *ShAmt; if (match(Op1, m_APInt(Mask))) { // If all bits in the inverted and shifted mask are clear: // and (shl X, ShAmt), Mask --> shl X, ShAmt if (match(Op0, m_Shl(m_Value(X), m_APInt(ShAmt))) && - (~(*Mask)).lshr(*ShAmt).isNullValue()) + (~(*Mask)).lshr(*ShAmt).isZero()) return Op0; // If all bits in the inverted and shifted mask are clear: // and (lshr X, ShAmt), Mask --> lshr X, ShAmt if (match(Op0, m_LShr(m_Value(X), m_APInt(ShAmt))) && - (~(*Mask)).shl(*ShAmt).isNullValue()) + (~(*Mask)).shl(*ShAmt).isZero()) return Op0; } @@ -2141,7 +2149,7 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, // if Mask = ((1 << effective_width_of(X)) - 1) << A // SimplifyDemandedBits in InstCombine can optimize the general case. // This pattern aims to help other passes for a common case. - Value *Y, *XShifted; + Value *XShifted; if (match(Op1, m_APInt(Mask)) && match(Op0, m_c_Or(m_CombineAnd(m_NUWShl(m_Value(X), m_APInt(ShAmt)), m_Value(XShifted)), @@ -2149,11 +2157,11 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, const unsigned Width = Op0->getType()->getScalarSizeInBits(); const unsigned ShftCnt = ShAmt->getLimitedValue(Width); const KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); - const unsigned EffWidthY = Width - YKnown.countMinLeadingZeros(); + const unsigned EffWidthY = YKnown.countMaxActiveBits(); if (EffWidthY <= ShftCnt) { const KnownBits XKnown = computeKnownBits(X, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); - const unsigned EffWidthX = Width - XKnown.countMinLeadingZeros(); + const unsigned EffWidthX = XKnown.countMaxActiveBits(); const APInt EffBitsY = APInt::getLowBitsSet(Width, EffWidthY); const APInt EffBitsX = APInt::getLowBitsSet(Width, EffWidthX) << ShftCnt; // If the mask is extracting all bits from X or Y as is, we can skip @@ -2257,6 +2265,19 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, match(Op0, m_c_Xor(m_Not(m_Specific(A)), m_Specific(B))))) return Op0; + // (A | B) | (A ^ B) --> A | B + // (B | A) | (A ^ B) --> B | A + if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && + match(Op0, m_c_Or(m_Specific(A), m_Specific(B)))) + return Op0; + + // Commute the outer 'or' operands. + // (A ^ B) | (A | B) --> A | B + // (A ^ B) | (B | A) --> B | A + if (match(Op0, m_Xor(m_Value(A), m_Value(B))) && + match(Op1, m_c_Or(m_Specific(A), m_Specific(B)))) + return Op1; + // (~A & B) | ~(A | B) --> ~A // (~A & B) | ~(B | A) --> ~A // (B & ~A) | ~(A | B) --> ~A @@ -2276,6 +2297,23 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, match(Op0, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) return NotA; + // Rotated -1 is still -1: + // (-1 << X) | (-1 >> (C - X)) --> -1 + // (-1 >> X) | (-1 << (C - X)) --> -1 + // ...with C <= bitwidth (and commuted variants). + Value *X, *Y; + if ((match(Op0, m_Shl(m_AllOnes(), m_Value(X))) && + match(Op1, m_LShr(m_AllOnes(), m_Value(Y)))) || + (match(Op1, m_Shl(m_AllOnes(), m_Value(X))) && + match(Op0, m_LShr(m_AllOnes(), m_Value(Y))))) { + const APInt *C; + if ((match(X, m_Sub(m_APInt(C), m_Specific(Y))) || + match(Y, m_Sub(m_APInt(C), m_Specific(X)))) && + C->ule(X->getType()->getScalarSizeInBits())) { + return ConstantInt::getAllOnesValue(X->getType()); + } + } + if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, false)) return V; @@ -3090,7 +3128,7 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS, // - C isn't zero. if (Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO)) || Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO)) || - match(LHS, m_Shl(m_One(), m_Value())) || !C->isNullValue()) { + match(LHS, m_Shl(m_One(), m_Value())) || !C->isZero()) { if (Pred == ICmpInst::ICMP_EQ) return ConstantInt::getFalse(GetCompareTy(RHS)); if (Pred == ICmpInst::ICMP_NE) @@ -3640,30 +3678,6 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, CRHS->getPointerOperand(), Q)) return C; - if (GetElementPtrInst *GLHS = dyn_cast<GetElementPtrInst>(LHS)) { - if (GEPOperator *GRHS = dyn_cast<GEPOperator>(RHS)) { - if (GLHS->getPointerOperand() == GRHS->getPointerOperand() && - GLHS->hasAllConstantIndices() && GRHS->hasAllConstantIndices() && - (ICmpInst::isEquality(Pred) || - (GLHS->isInBounds() && GRHS->isInBounds() && - Pred == ICmpInst::getSignedPredicate(Pred)))) { - // The bases are equal and the indices are constant. Build a constant - // expression GEP with the same indices and a null base pointer to see - // what constant folding can make out of it. - Constant *Null = Constant::getNullValue(GLHS->getPointerOperandType()); - SmallVector<Value *, 4> IndicesLHS(GLHS->indices()); - Constant *NewLHS = ConstantExpr::getGetElementPtr( - GLHS->getSourceElementType(), Null, IndicesLHS); - - SmallVector<Value *, 4> IndicesRHS(GRHS->idx_begin(), GRHS->idx_end()); - Constant *NewRHS = ConstantExpr::getGetElementPtr( - GLHS->getSourceElementType(), Null, IndicesRHS); - Constant *NewICmp = ConstantExpr::getICmp(Pred, NewLHS, NewRHS); - return ConstantFoldConstant(NewICmp, Q.DL); - } - } - } - // If the comparison is with the result of a select instruction, check whether // comparing with either branch of the select always yields the same value. if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS)) @@ -3966,7 +3980,8 @@ static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp, if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) return PreventSelfSimplify(SimplifyGEPInst(GEP->getSourceElementType(), - NewOps, Q, MaxRecurse - 1)); + NewOps, GEP->isInBounds(), Q, + MaxRecurse - 1)); if (isa<SelectInst>(I)) return PreventSelfSimplify( @@ -4080,6 +4095,22 @@ static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal, std::swap(TrueVal, FalseVal); } + // Check for integer min/max with a limit constant: + // X > MIN_INT ? X : MIN_INT --> X + // X < MAX_INT ? X : MAX_INT --> X + if (TrueVal->getType()->isIntOrIntVectorTy()) { + Value *X, *Y; + SelectPatternFlavor SPF = + matchDecomposedSelectPattern(cast<ICmpInst>(CondVal), TrueVal, FalseVal, + X, Y).Flavor; + if (SelectPatternResult::isMinOrMax(SPF) && Pred == getMinMaxPred(SPF)) { + APInt LimitC = getMinMaxLimit(getInverseMinMaxFlavor(SPF), + X->getType()->getScalarSizeInBits()); + if (match(Y, m_SpecificInt(LimitC))) + return X; + } + } + if (Pred == ICmpInst::ICMP_EQ && match(CmpRHS, m_Zero())) { Value *X; const APInt *Y; @@ -4210,14 +4241,27 @@ static Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, return FalseVal; } - // select i1 Cond, i1 true, i1 false --> i1 Cond assert(Cond->getType()->isIntOrIntVectorTy(1) && "Select must have bool or bool vector condition"); assert(TrueVal->getType() == FalseVal->getType() && "Select must have same types for true/false ops"); - if (Cond->getType() == TrueVal->getType() && - match(TrueVal, m_One()) && match(FalseVal, m_ZeroInt())) - return Cond; + + if (Cond->getType() == TrueVal->getType()) { + // select i1 Cond, i1 true, i1 false --> i1 Cond + if (match(TrueVal, m_One()) && match(FalseVal, m_ZeroInt())) + return Cond; + + // (X || Y) && (X || !Y) --> X (commuted 8 ways) + Value *X, *Y; + if (match(FalseVal, m_ZeroInt())) { + if (match(Cond, m_c_LogicalOr(m_Value(X), m_Not(m_Value(Y)))) && + match(TrueVal, m_c_LogicalOr(m_Specific(X), m_Specific(Y)))) + return X; + if (match(TrueVal, m_c_LogicalOr(m_Value(X), m_Not(m_Value(Y)))) && + match(Cond, m_c_LogicalOr(m_Specific(X), m_Specific(Y)))) + return X; + } + } // select ?, X, X -> X if (TrueVal == FalseVal) @@ -4295,7 +4339,7 @@ Value *llvm::SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, /// Given operands for an GetElementPtrInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, +static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, bool InBounds, const SimplifyQuery &Q, unsigned) { // The type of the GEP pointer operand. unsigned AS = @@ -4396,14 +4440,14 @@ static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, // gep (gep V, C), (sub 0, V) -> C if (match(Ops.back(), m_Sub(m_Zero(), m_PtrToInt(m_Specific(StrippedBasePtr)))) && - !BasePtrOffset.isNullValue()) { + !BasePtrOffset.isZero()) { auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset); return ConstantExpr::getIntToPtr(CI, GEPTy); } // gep (gep V, C), (xor V, -1) -> C-1 if (match(Ops.back(), m_Xor(m_PtrToInt(m_Specific(StrippedBasePtr)), m_AllOnes())) && - !BasePtrOffset.isOneValue()) { + !BasePtrOffset.isOne()) { auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset - 1); return ConstantExpr::getIntToPtr(CI, GEPTy); } @@ -4415,13 +4459,13 @@ static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, return nullptr; auto *CE = ConstantExpr::getGetElementPtr(SrcTy, cast<Constant>(Ops[0]), - Ops.slice(1)); + Ops.slice(1), InBounds); return ConstantFoldConstant(CE, Q.DL); } -Value *llvm::SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, +Value *llvm::SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, bool InBounds, const SimplifyQuery &Q) { - return ::SimplifyGEPInst(SrcTy, Ops, Q, RecursionLimit); + return ::SimplifyGEPInst(SrcTy, Ops, InBounds, Q, RecursionLimit); } /// Given operands for an InsertValueInst, see if we can fold the result. @@ -4891,6 +4935,11 @@ static Constant *simplifyFPOp(ArrayRef<Value *> Ops, FastMathFlags FMF, return nullptr; } +// TODO: Move this out to a header file: +static inline bool canIgnoreSNaN(fp::ExceptionBehavior EB, FastMathFlags FMF) { + return (EB == fp::ebIgnore || FMF.noNaNs()); +} + /// Given operands for an FAdd, see if we can fold the result. If not, this /// returns null. static Value * @@ -4905,17 +4954,25 @@ SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) return C; - if (!isDefaultFPEnvironment(ExBehavior, Rounding)) - return nullptr; - // fadd X, -0 ==> X - if (match(Op1, m_NegZeroFP())) - return Op0; + // With strict/constrained FP, we have these possible edge cases that do + // not simplify to Op0: + // fadd SNaN, -0.0 --> QNaN + // fadd +0.0, -0.0 --> -0.0 (but only with round toward negative) + if (canIgnoreSNaN(ExBehavior, FMF) && + (!canRoundingModeBe(Rounding, RoundingMode::TowardNegative) || + FMF.noSignedZeros())) + if (match(Op1, m_NegZeroFP())) + return Op0; // fadd X, 0 ==> X, when we know X is not -0 - if (match(Op1, m_PosZeroFP()) && - (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) - return Op0; + if (canIgnoreSNaN(ExBehavior, FMF)) + if (match(Op1, m_PosZeroFP()) && + (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) + return Op0; + + if (!isDefaultFPEnvironment(ExBehavior, Rounding)) + return nullptr; // With nnan: -X + X --> 0.0 (and commuted variant) // We don't have to explicitly exclude infinities (ninf): INF + -INF == NaN. @@ -5457,6 +5514,9 @@ static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0, if (match(Op0, m_Intrinsic<Intrinsic::experimental_vector_reverse>(m_Value(X)))) return X; + // experimental.vector.reverse(splat(X)) -> splat(X) + if (isSplatValue(Op0)) + return Op0; break; default: break; @@ -5772,13 +5832,32 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1, static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) { - // Intrinsics with no operands have some kind of side effect. Don't simplify. - unsigned NumOperands = Call->getNumArgOperands(); - if (!NumOperands) - return nullptr; - + unsigned NumOperands = Call->arg_size(); Function *F = cast<Function>(Call->getCalledFunction()); Intrinsic::ID IID = F->getIntrinsicID(); + + // Most of the intrinsics with no operands have some kind of side effect. + // Don't simplify. + if (!NumOperands) { + switch (IID) { + case Intrinsic::vscale: { + // Call may not be inserted into the IR yet at point of calling simplify. + if (!Call->getParent() || !Call->getParent()->getParent()) + return nullptr; + auto Attr = Call->getFunction()->getFnAttribute(Attribute::VScaleRange); + if (!Attr.isValid()) + return nullptr; + unsigned VScaleMin, VScaleMax; + std::tie(VScaleMin, VScaleMax) = Attr.getVScaleRangeArgs(); + if (VScaleMin == VScaleMax && VScaleMax != 0) + return ConstantInt::get(F->getReturnType(), VScaleMin); + return nullptr; + } + default: + return nullptr; + } + } + if (NumOperands == 1) return simplifyUnaryIntrinsic(F, Call->getArgOperand(0), Q); @@ -5814,9 +5893,18 @@ static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) { if (match(ShAmtArg, m_APInt(ShAmtC))) { // If there's effectively no shift, return the 1st arg or 2nd arg. APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth()); - if (ShAmtC->urem(BitWidth).isNullValue()) + if (ShAmtC->urem(BitWidth).isZero()) return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); } + + // Rotating zero by anything is zero. + if (match(Op0, m_Zero()) && match(Op1, m_Zero())) + return ConstantInt::getNullValue(F->getReturnType()); + + // Rotating -1 by anything is -1. + if (match(Op0, m_AllOnes()) && match(Op1, m_AllOnes())) + return ConstantInt::getAllOnesValue(F->getReturnType()); + return nullptr; } case Intrinsic::experimental_constrained_fma: { @@ -5939,7 +6027,7 @@ static Value *tryConstantFoldCall(CallBase *Call, const SimplifyQuery &Q) { return nullptr; SmallVector<Constant *, 4> ConstantArgs; - unsigned NumArgs = Call->getNumArgOperands(); + unsigned NumArgs = Call->arg_size(); ConstantArgs.reserve(NumArgs); for (auto &Arg : Call->args()) { Constant *C = dyn_cast<Constant>(&Arg); @@ -5990,73 +6078,27 @@ Value *llvm::SimplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { return ::SimplifyFreezeInst(Op0, Q); } -static Constant *ConstructLoadOperandConstant(Value *Op) { - SmallVector<Value *, 4> Worklist; - // Invalid IR in unreachable code may contain self-referential values. Don't infinitely loop. - SmallPtrSet<Value *, 4> Visited; - Worklist.push_back(Op); - while (true) { - Value *CurOp = Worklist.back(); - if (!Visited.insert(CurOp).second) - return nullptr; - if (isa<Constant>(CurOp)) - break; - if (auto *BC = dyn_cast<BitCastOperator>(CurOp)) { - Worklist.push_back(BC->getOperand(0)); - } else if (auto *GEP = dyn_cast<GEPOperator>(CurOp)) { - for (unsigned I = 1; I != GEP->getNumOperands(); ++I) { - if (!isa<Constant>(GEP->getOperand(I))) - return nullptr; - } - Worklist.push_back(GEP->getOperand(0)); - } else if (auto *II = dyn_cast<IntrinsicInst>(CurOp)) { - if (II->isLaunderOrStripInvariantGroup()) - Worklist.push_back(II->getOperand(0)); - else - return nullptr; - } else { - return nullptr; - } - } - - Constant *NewOp = cast<Constant>(Worklist.pop_back_val()); - while (!Worklist.empty()) { - Value *CurOp = Worklist.pop_back_val(); - if (isa<BitCastOperator>(CurOp)) { - NewOp = ConstantExpr::getBitCast(NewOp, CurOp->getType()); - } else if (auto *GEP = dyn_cast<GEPOperator>(CurOp)) { - SmallVector<Constant *> Idxs; - Idxs.reserve(GEP->getNumOperands() - 1); - for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) { - Idxs.push_back(cast<Constant>(GEP->getOperand(I))); - } - NewOp = ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), NewOp, - Idxs, GEP->isInBounds(), - GEP->getInRangeIndex()); - } else { - assert(isa<IntrinsicInst>(CurOp) && - cast<IntrinsicInst>(CurOp)->isLaunderOrStripInvariantGroup() && - "expected invariant group intrinsic"); - NewOp = ConstantExpr::getBitCast(NewOp, CurOp->getType()); - } - } - return NewOp; -} - static Value *SimplifyLoadInst(LoadInst *LI, Value *PtrOp, const SimplifyQuery &Q) { if (LI->isVolatile()) return nullptr; - // Try to make the load operand a constant, specifically handle - // invariant.group intrinsics. + APInt Offset(Q.DL.getIndexTypeSizeInBits(PtrOp->getType()), 0); auto *PtrOpC = dyn_cast<Constant>(PtrOp); - if (!PtrOpC) - PtrOpC = ConstructLoadOperandConstant(PtrOp); + // Try to convert operand into a constant by stripping offsets while looking + // through invariant.group intrinsics. Don't bother if the underlying object + // is not constant, as calculating GEP offsets is expensive. + if (!PtrOpC && isa<Constant>(getUnderlyingObject(PtrOp))) { + PtrOp = PtrOp->stripAndAccumulateConstantOffsets( + Q.DL, Offset, /* AllowNonInbounts */ true, + /* AllowInvariantGroup */ true); + // Index size may have changed due to address space casts. + Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType())); + PtrOpC = dyn_cast<Constant>(PtrOp); + } if (PtrOpC) - return ConstantFoldLoadFromConstPtr(PtrOpC, LI->getType(), Q.DL); - + return ConstantFoldLoadFromConstPtr(PtrOpC, LI->getType(), Offset, Q.DL); return nullptr; } @@ -6156,8 +6198,9 @@ static Value *simplifyInstructionWithOperands(Instruction *I, Result = SimplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q); break; case Instruction::GetElementPtr: { - Result = SimplifyGEPInst(cast<GetElementPtrInst>(I)->getSourceElementType(), - NewOps, Q); + auto *GEPI = cast<GetElementPtrInst>(I); + Result = SimplifyGEPInst(GEPI->getSourceElementType(), NewOps, + GEPI->isInBounds(), Q); break; } case Instruction::InsertValue: { diff --git a/llvm/lib/Analysis/LazyCallGraph.cpp b/llvm/lib/Analysis/LazyCallGraph.cpp index 8f87552fca1f..0007c54b16d0 100644 --- a/llvm/lib/Analysis/LazyCallGraph.cpp +++ b/llvm/lib/Analysis/LazyCallGraph.cpp @@ -220,8 +220,7 @@ bool LazyCallGraph::invalidate(Module &, const PreservedAnalyses &PA, // Check whether the analysis, all analyses on functions, or the function's // CFG have been preserved. auto PAC = PA.getChecker<llvm::LazyCallGraphAnalysis>(); - return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>() || - PAC.preservedSet<CFGAnalyses>()); + return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()); } LazyCallGraph &LazyCallGraph::operator=(LazyCallGraph &&G) { @@ -1962,6 +1961,29 @@ void LazyCallGraph::buildRefSCCs() { }); } +void LazyCallGraph::visitReferences(SmallVectorImpl<Constant *> &Worklist, + SmallPtrSetImpl<Constant *> &Visited, + function_ref<void(Function &)> Callback) { + while (!Worklist.empty()) { + Constant *C = Worklist.pop_back_val(); + + if (Function *F = dyn_cast<Function>(C)) { + if (!F->isDeclaration()) + Callback(*F); + continue; + } + + // blockaddresses are weird and don't participate in the call graph anyway, + // skip them. + if (isa<BlockAddress>(C)) + continue; + + for (Value *Op : C->operand_values()) + if (Visited.insert(cast<Constant>(Op)).second) + Worklist.push_back(cast<Constant>(Op)); + } +} + AnalysisKey LazyCallGraphAnalysis::Key; LazyCallGraphPrinterPass::LazyCallGraphPrinterPass(raw_ostream &OS) : OS(OS) {} diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp index 1dababafb8a6..50fa169c2081 100644 --- a/llvm/lib/Analysis/LazyValueInfo.cpp +++ b/llvm/lib/Analysis/LazyValueInfo.cpp @@ -126,7 +126,7 @@ static ValueLatticeElement intersect(const ValueLatticeElement &A, // Note: An empty range is implicitly converted to unknown or undef depending // on MayIncludeUndef internally. return ValueLatticeElement::getRange( - std::move(Range), /*MayIncludeUndef=*/A.isConstantRangeIncludingUndef() | + std::move(Range), /*MayIncludeUndef=*/A.isConstantRangeIncludingUndef() || B.isConstantRangeIncludingUndef()); } @@ -832,7 +832,7 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueSelect( }; }(); return ValueLatticeElement::getRange( - ResultCR, TrueVal.isConstantRangeIncludingUndef() | + ResultCR, TrueVal.isConstantRangeIncludingUndef() || FalseVal.isConstantRangeIncludingUndef()); } @@ -846,7 +846,7 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueSelect( } if (SPR.Flavor == SPF_NABS) { - ConstantRange Zero(APInt::getNullValue(TrueCR.getBitWidth())); + ConstantRange Zero(APInt::getZero(TrueCR.getBitWidth())); if (LHS == SI->getTrueValue()) return ValueLatticeElement::getRange( Zero.sub(TrueCR.abs()), FalseVal.isConstantRangeIncludingUndef()); @@ -1117,12 +1117,11 @@ static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI, } // If (Val & Mask) != 0 then the value must be larger than the lowest set // bit of Mask. - if (EdgePred == ICmpInst::ICMP_NE && !Mask->isNullValue() && - C->isNullValue()) { + if (EdgePred == ICmpInst::ICMP_NE && !Mask->isZero() && C->isZero()) { unsigned BitWidth = Ty->getIntegerBitWidth(); return ValueLatticeElement::getRange(ConstantRange::getNonEmpty( APInt::getOneBitSet(BitWidth, Mask->countTrailingZeros()), - APInt::getNullValue(BitWidth))); + APInt::getZero(BitWidth))); } } @@ -1780,62 +1779,62 @@ LazyValueInfo::getPredicateAt(unsigned Pred, Value *V, Constant *C, // We could consider extending this to search further backwards through the // CFG and/or value graph, but there are non-obvious compile time vs quality // tradeoffs. - if (CxtI) { - BasicBlock *BB = CxtI->getParent(); + BasicBlock *BB = CxtI->getParent(); - // Function entry or an unreachable block. Bail to avoid confusing - // analysis below. - pred_iterator PI = pred_begin(BB), PE = pred_end(BB); - if (PI == PE) - return Unknown; + // Function entry or an unreachable block. Bail to avoid confusing + // analysis below. + pred_iterator PI = pred_begin(BB), PE = pred_end(BB); + if (PI == PE) + return Unknown; - // If V is a PHI node in the same block as the context, we need to ask - // questions about the predicate as applied to the incoming value along - // each edge. This is useful for eliminating cases where the predicate is - // known along all incoming edges. - if (auto *PHI = dyn_cast<PHINode>(V)) - if (PHI->getParent() == BB) { - Tristate Baseline = Unknown; - for (unsigned i = 0, e = PHI->getNumIncomingValues(); i < e; i++) { - Value *Incoming = PHI->getIncomingValue(i); - BasicBlock *PredBB = PHI->getIncomingBlock(i); - // Note that PredBB may be BB itself. - Tristate Result = getPredicateOnEdge(Pred, Incoming, C, PredBB, BB, - CxtI); + // If V is a PHI node in the same block as the context, we need to ask + // questions about the predicate as applied to the incoming value along + // each edge. This is useful for eliminating cases where the predicate is + // known along all incoming edges. + if (auto *PHI = dyn_cast<PHINode>(V)) + if (PHI->getParent() == BB) { + Tristate Baseline = Unknown; + for (unsigned i = 0, e = PHI->getNumIncomingValues(); i < e; i++) { + Value *Incoming = PHI->getIncomingValue(i); + BasicBlock *PredBB = PHI->getIncomingBlock(i); + // Note that PredBB may be BB itself. + Tristate Result = + getPredicateOnEdge(Pred, Incoming, C, PredBB, BB, CxtI); - // Keep going as long as we've seen a consistent known result for - // all inputs. - Baseline = (i == 0) ? Result /* First iteration */ - : (Baseline == Result ? Baseline : Unknown); /* All others */ - if (Baseline == Unknown) - break; - } - if (Baseline != Unknown) - return Baseline; + // Keep going as long as we've seen a consistent known result for + // all inputs. + Baseline = (i == 0) ? Result /* First iteration */ + : (Baseline == Result ? Baseline + : Unknown); /* All others */ + if (Baseline == Unknown) + break; } + if (Baseline != Unknown) + return Baseline; + } - // For a comparison where the V is outside this block, it's possible - // that we've branched on it before. Look to see if the value is known - // on all incoming edges. - if (!isa<Instruction>(V) || - cast<Instruction>(V)->getParent() != BB) { - // For predecessor edge, determine if the comparison is true or false - // on that edge. If they're all true or all false, we can conclude - // the value of the comparison in this block. - Tristate Baseline = getPredicateOnEdge(Pred, V, C, *PI, BB, CxtI); - if (Baseline != Unknown) { - // Check that all remaining incoming values match the first one. - while (++PI != PE) { - Tristate Ret = getPredicateOnEdge(Pred, V, C, *PI, BB, CxtI); - if (Ret != Baseline) break; - } - // If we terminated early, then one of the values didn't match. - if (PI == PE) { - return Baseline; - } + // For a comparison where the V is outside this block, it's possible + // that we've branched on it before. Look to see if the value is known + // on all incoming edges. + if (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB) { + // For predecessor edge, determine if the comparison is true or false + // on that edge. If they're all true or all false, we can conclude + // the value of the comparison in this block. + Tristate Baseline = getPredicateOnEdge(Pred, V, C, *PI, BB, CxtI); + if (Baseline != Unknown) { + // Check that all remaining incoming values match the first one. + while (++PI != PE) { + Tristate Ret = getPredicateOnEdge(Pred, V, C, *PI, BB, CxtI); + if (Ret != Baseline) + break; + } + // If we terminated early, then one of the values didn't match. + if (PI == PE) { + return Baseline; } } } + return Unknown; } diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp index 4de5e1e06c7e..f9a7a5bdf434 100644 --- a/llvm/lib/Analysis/Lint.cpp +++ b/llvm/lib/Analysis/Lint.cpp @@ -235,7 +235,7 @@ void Lint::visitCallBase(CallBase &I) { for (auto BI = I.arg_begin(); BI != AE; ++BI, ++ArgNo) { // Skip ByVal arguments since they will be memcpy'd to the callee's // stack so we're not really passing the pointer anyway. - if (PAL.hasParamAttribute(ArgNo, Attribute::ByVal)) + if (PAL.hasParamAttr(ArgNo, Attribute::ByVal)) continue; // If both arguments are readonly, they have no dependence. if (Formal->onlyReadsMemory() && I.onlyReadsMemory(ArgNo)) @@ -268,7 +268,7 @@ void Lint::visitCallBase(CallBase &I) { for (Value *Arg : I.args()) { // Skip ByVal arguments since they will be memcpy'd to the callee's // stack anyway. - if (PAL.hasParamAttribute(ArgNo++, Attribute::ByVal)) + if (PAL.hasParamAttr(ArgNo++, Attribute::ByVal)) continue; Value *Obj = findValue(Arg, /*OffsetOk=*/true); Assert(!isa<AllocaInst>(Obj), @@ -715,6 +715,7 @@ PreservedAnalyses LintPass::run(Function &F, FunctionAnalysisManager &AM) { return PreservedAnalyses::all(); } +namespace { class LintLegacyPass : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid @@ -733,6 +734,7 @@ public: } void print(raw_ostream &O, const Module *M) const override {} }; +} // namespace char LintLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN(LintLegacyPass, "lint", "Statically lint-checks LLVM IR", diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index 1c55f485aa76..0fbf1db0685d 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -147,7 +147,7 @@ static bool isDereferenceableAndAlignedPointer( Alignment, Size, DL, CtxI, DT, TLI, Visited, MaxDepth); - if (const AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(V)) + if (const AddrSpaceCastOperator *ASC = dyn_cast<AddrSpaceCastOperator>(V)) return isDereferenceableAndAlignedPointer(ASC->getOperand(0), Alignment, Size, DL, CtxI, DT, TLI, Visited, MaxDepth); @@ -451,8 +451,8 @@ static bool areNonOverlapSameBaseLoadAndStore(const Value *LoadPtr, const Value *StorePtr, Type *StoreTy, const DataLayout &DL) { - APInt LoadOffset(DL.getTypeSizeInBits(LoadPtr->getType()), 0); - APInt StoreOffset(DL.getTypeSizeInBits(StorePtr->getType()), 0); + APInt LoadOffset(DL.getIndexTypeSizeInBits(LoadPtr->getType()), 0); + APInt StoreOffset(DL.getIndexTypeSizeInBits(StorePtr->getType()), 0); const Value *LoadBase = LoadPtr->stripAndAccumulateConstantOffsets( DL, LoadOffset, /* AllowNonInbounds */ false); const Value *StoreBase = StorePtr->stripAndAccumulateConstantOffsets( @@ -511,8 +511,11 @@ static Value *getAvailableLoadStore(Instruction *Inst, const Value *Ptr, if (CastInst::isBitOrNoopPointerCastable(Val->getType(), AccessTy, DL)) return Val; - if (auto *C = dyn_cast<Constant>(Val)) - return ConstantFoldLoadThroughBitcast(C, AccessTy, DL); + TypeSize StoreSize = DL.getTypeStoreSize(Val->getType()); + TypeSize LoadSize = DL.getTypeStoreSize(AccessTy); + if (TypeSize::isKnownLE(LoadSize, StoreSize)) + if (auto *C = dyn_cast<Constant>(Val)) + return ConstantFoldLoadFromConst(C, AccessTy, DL); } return nullptr; diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index a239928ecf38..f9bd7167317f 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -142,13 +142,12 @@ Value *llvm::stripIntegerCast(Value *V) { const SCEV *llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const ValueToValueMap &PtrToStride, - Value *Ptr, Value *OrigPtr) { + Value *Ptr) { const SCEV *OrigSCEV = PSE.getSCEV(Ptr); // If there is an entry in the map return the SCEV of the pointer with the // symbolic stride replaced by one. - ValueToValueMap::const_iterator SI = - PtrToStride.find(OrigPtr ? OrigPtr : Ptr); + ValueToValueMap::const_iterator SI = PtrToStride.find(Ptr); if (SI == PtrToStride.end()) // For a non-symbolic stride, just return the original expression. return OrigSCEV; @@ -659,7 +658,8 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, if (PSE.getSE()->isLoopInvariant(PtrScev, L)) return true; - int64_t Stride = getPtrStride(PSE, Ptr, L, Strides); + Type *AccessTy = Ptr->getType()->getPointerElementType(); + int64_t Stride = getPtrStride(PSE, AccessTy, Ptr, L, Strides); if (Stride == 1 || PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW)) return true; @@ -1026,15 +1026,17 @@ static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR, } /// Check whether the access through \p Ptr has a constant stride. -int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, - const Loop *Lp, const ValueToValueMap &StridesMap, - bool Assume, bool ShouldCheckWrap) { +int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, + Value *Ptr, const Loop *Lp, + const ValueToValueMap &StridesMap, bool Assume, + bool ShouldCheckWrap) { Type *Ty = Ptr->getType(); assert(Ty->isPointerTy() && "Unexpected non-ptr"); + unsigned AddrSpace = Ty->getPointerAddressSpace(); - // Make sure that the pointer does not point to aggregate types. - auto *PtrTy = cast<PointerType>(Ty); - if (PtrTy->getElementType()->isAggregateType()) { + // Make sure we're not accessing an aggregate type. + // TODO: Why? This doesn't make any sense. + if (AccessTy->isAggregateType()) { LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not a pointer to a scalar type" << *Ptr << "\n"); return 0; @@ -1071,8 +1073,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW) || isNoWrapAddRec(Ptr, AR, PSE, Lp); if (!IsNoWrapAddRec && !IsInBoundsGEP && - NullPointerIsDefined(Lp->getHeader()->getParent(), - PtrTy->getAddressSpace())) { + NullPointerIsDefined(Lp->getHeader()->getParent(), AddrSpace)) { if (Assume) { PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW); IsNoWrapAddRec = true; @@ -1100,7 +1101,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, } auto &DL = Lp->getHeader()->getModule()->getDataLayout(); - int64_t Size = DL.getTypeAllocSize(PtrTy->getElementType()); + int64_t Size = DL.getTypeAllocSize(AccessTy); const APInt &APStepVal = C->getAPInt(); // Huge step value - give up. @@ -1120,7 +1121,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, // zero we know that this won't happen without triggering undefined behavior. if (!IsNoWrapAddRec && Stride != 1 && Stride != -1 && (IsInBoundsGEP || !NullPointerIsDefined(Lp->getHeader()->getParent(), - PtrTy->getAddressSpace()))) { + AddrSpace))) { if (Assume) { // We can avoid this case by adding a run-time check. LLVM_DEBUG(dbgs() << "LAA: Non unit strided pointer which is not either " @@ -1262,6 +1263,47 @@ bool llvm::isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, return Diff && *Diff == 1; } +static void visitPointers(Value *StartPtr, const Loop &InnermostLoop, + function_ref<void(Value *)> AddPointer) { + SmallPtrSet<Value *, 8> Visited; + SmallVector<Value *> WorkList; + WorkList.push_back(StartPtr); + + while (!WorkList.empty()) { + Value *Ptr = WorkList.pop_back_val(); + if (!Visited.insert(Ptr).second) + continue; + auto *PN = dyn_cast<PHINode>(Ptr); + // SCEV does not look through non-header PHIs inside the loop. Such phis + // can be analyzed by adding separate accesses for each incoming pointer + // value. + if (PN && InnermostLoop.contains(PN->getParent()) && + PN->getParent() != InnermostLoop.getHeader()) { + for (const Use &Inc : PN->incoming_values()) + WorkList.push_back(Inc); + } else + AddPointer(Ptr); + } +} + +void MemoryDepChecker::addAccess(StoreInst *SI) { + visitPointers(SI->getPointerOperand(), *InnermostLoop, + [this, SI](Value *Ptr) { + Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx); + InstMap.push_back(SI); + ++AccessIdx; + }); +} + +void MemoryDepChecker::addAccess(LoadInst *LI) { + visitPointers(LI->getPointerOperand(), *InnermostLoop, + [this, LI](Value *Ptr) { + Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx); + InstMap.push_back(LI); + ++AccessIdx; + }); +} + MemoryDepChecker::VectorizationSafetyStatus MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) { switch (Type) { @@ -1478,6 +1520,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, Value *BPtr = B.getPointer(); bool AIsWrite = A.getInt(); bool BIsWrite = B.getInt(); + Type *ATy = APtr->getType()->getPointerElementType(); + Type *BTy = BPtr->getType()->getPointerElementType(); // Two reads are independent. if (!AIsWrite && !BIsWrite) @@ -1488,8 +1532,10 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, BPtr->getType()->getPointerAddressSpace()) return Dependence::Unknown; - int64_t StrideAPtr = getPtrStride(PSE, APtr, InnermostLoop, Strides, true); - int64_t StrideBPtr = getPtrStride(PSE, BPtr, InnermostLoop, Strides, true); + int64_t StrideAPtr = + getPtrStride(PSE, ATy, APtr, InnermostLoop, Strides, true); + int64_t StrideBPtr = + getPtrStride(PSE, BTy, BPtr, InnermostLoop, Strides, true); const SCEV *Src = PSE.getSCEV(APtr); const SCEV *Sink = PSE.getSCEV(BPtr); @@ -1498,6 +1544,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, // dependence. if (StrideAPtr < 0) { std::swap(APtr, BPtr); + std::swap(ATy, BTy); std::swap(Src, Sink); std::swap(AIsWrite, BIsWrite); std::swap(AIdx, BIdx); @@ -1519,8 +1566,6 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, return Dependence::Unknown; } - Type *ATy = APtr->getType()->getPointerElementType(); - Type *BTy = BPtr->getType()->getPointerElementType(); auto &DL = InnermostLoop->getHeader()->getModule()->getDataLayout(); uint64_t TypeByteSize = DL.getTypeAllocSize(ATy); uint64_t Stride = std::abs(StrideAPtr); @@ -1958,7 +2003,11 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI, if (blockNeedsPredication(ST->getParent(), TheLoop, DT)) Loc.AATags.TBAA = nullptr; - Accesses.addStore(Loc); + visitPointers(const_cast<Value *>(Loc.Ptr), *TheLoop, + [&Accesses, Loc](Value *Ptr) { + MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr); + Accesses.addStore(NewLoc); + }); } } @@ -1982,7 +2031,7 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI, // words may be written to the same address. bool IsReadOnlyPtr = false; if (Seen.insert(Ptr).second || - !getPtrStride(*PSE, Ptr, TheLoop, SymbolicStrides)) { + !getPtrStride(*PSE, LD->getType(), Ptr, TheLoop, SymbolicStrides)) { ++NumReads; IsReadOnlyPtr = true; } @@ -2002,7 +2051,11 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI, if (blockNeedsPredication(LD->getParent(), TheLoop, DT)) Loc.AATags.TBAA = nullptr; - Accesses.addLoad(Loc, IsReadOnlyPtr); + visitPointers(const_cast<Value *>(Loc.Ptr), *TheLoop, + [&Accesses, Loc, IsReadOnlyPtr](Value *Ptr) { + MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr); + Accesses.addLoad(NewLoc, IsReadOnlyPtr); + }); } // If we write (or read-write) to a single destination and there are no diff --git a/llvm/lib/Analysis/LoopCacheAnalysis.cpp b/llvm/lib/Analysis/LoopCacheAnalysis.cpp index 8a613647bbea..7b895d8a5dc2 100644 --- a/llvm/lib/Analysis/LoopCacheAnalysis.cpp +++ b/llvm/lib/Analysis/LoopCacheAnalysis.cpp @@ -30,6 +30,7 @@ #include "llvm/ADT/Sequence.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/Delinearization.h" #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" @@ -290,8 +291,8 @@ CacheCostTy IndexedReference::computeRefCost(const Loop &L, const SCEV *Coeff = getLastCoefficient(); const SCEV *ElemSize = Sizes.back(); const SCEV *Stride = SE.getMulExpr(Coeff, ElemSize); - const SCEV *CacheLineSize = SE.getConstant(Stride->getType(), CLS); Type *WiderType = SE.getWiderType(Stride->getType(), TripCount->getType()); + const SCEV *CacheLineSize = SE.getConstant(WiderType, CLS); if (SE.isKnownNegative(Stride)) Stride = SE.getNegativeSCEV(Stride); Stride = SE.getNoopOrAnyExtend(Stride, WiderType); @@ -344,8 +345,8 @@ bool IndexedReference::delinearize(const LoopInfo &LI) { LLVM_DEBUG(dbgs().indent(2) << "In Loop '" << L->getName() << "', AccessFn: " << *AccessFn << "\n"); - SE.delinearize(AccessFn, Subscripts, Sizes, - SE.getElementSize(&StoreOrLoadInst)); + llvm::delinearize(SE, AccessFn, Subscripts, Sizes, + SE.getElementSize(&StoreOrLoadInst)); if (Subscripts.empty() || Sizes.empty() || Subscripts.size() != Sizes.size()) { @@ -425,9 +426,7 @@ bool IndexedReference::isConsecutive(const Loop &L, unsigned CLS) const { const SCEV *IndexedReference::getLastCoefficient() const { const SCEV *LastSubscript = getLastSubscript(); - assert(isa<SCEVAddRecExpr>(LastSubscript) && - "Expecting a SCEV add recurrence expression"); - const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LastSubscript); + auto *AR = cast<SCEVAddRecExpr>(LastSubscript); return AR->getStepRecurrence(SE); } @@ -522,10 +521,9 @@ void CacheCost::calculateCacheFootprint() { LLVM_DEBUG(dbgs() << "COMPUTING LOOP CACHE COSTS\n"); for (const Loop *L : Loops) { - assert((std::find_if(LoopCosts.begin(), LoopCosts.end(), - [L](const LoopCacheCostTy &LCC) { - return LCC.first == L; - }) == LoopCosts.end()) && + assert(llvm::none_of( + LoopCosts, + [L](const LoopCacheCostTy &LCC) { return LCC.first == L; }) && "Should not add duplicate element"); CacheCostTy LoopCost = computeLoopCacheCost(*L, RefGroups); LoopCosts.push_back(std::make_pair(L, LoopCost)); diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp index 66aab4c195c8..b35fb2a190f6 100644 --- a/llvm/lib/Analysis/LoopInfo.cpp +++ b/llvm/lib/Analysis/LoopInfo.cpp @@ -301,15 +301,16 @@ PHINode *Loop::getInductionVariable(ScalarEvolution &SE) const { if (!CmpInst) return nullptr; - Instruction *LatchCmpOp0 = dyn_cast<Instruction>(CmpInst->getOperand(0)); - Instruction *LatchCmpOp1 = dyn_cast<Instruction>(CmpInst->getOperand(1)); + Value *LatchCmpOp0 = CmpInst->getOperand(0); + Value *LatchCmpOp1 = CmpInst->getOperand(1); for (PHINode &IndVar : Header->phis()) { InductionDescriptor IndDesc; if (!InductionDescriptor::isInductionPHI(&IndVar, this, &SE, IndDesc)) continue; - Instruction *StepInst = IndDesc.getInductionBinOp(); + BasicBlock *Latch = getLoopLatch(); + Value *StepInst = IndVar.getIncomingValueForBlock(Latch); // case 1: // IndVar = phi[{InitialValue, preheader}, {StepInst, latch}] @@ -1102,6 +1103,11 @@ llvm::Optional<int> llvm::getOptionalIntLoopAttribute(const Loop *TheLoop, return IntMD->getSExtValue(); } +int llvm::getIntLoopAttribute(const Loop *TheLoop, StringRef Name, + int Default) { + return getOptionalIntLoopAttribute(TheLoop, Name).getValueOr(Default); +} + static const char *LLVMLoopMustProgress = "llvm.loop.mustprogress"; bool llvm::hasMustProgress(const Loop *L) { diff --git a/llvm/lib/Analysis/LoopNestAnalysis.cpp b/llvm/lib/Analysis/LoopNestAnalysis.cpp index 2649ed60f762..675bb7a7749c 100644 --- a/llvm/lib/Analysis/LoopNestAnalysis.cpp +++ b/llvm/lib/Analysis/LoopNestAnalysis.cpp @@ -50,8 +50,66 @@ std::unique_ptr<LoopNest> LoopNest::getLoopNest(Loop &Root, return std::make_unique<LoopNest>(Root, SE); } +static CmpInst *getOuterLoopLatchCmp(const Loop &OuterLoop) { + + const BasicBlock *Latch = OuterLoop.getLoopLatch(); + assert(Latch && "Expecting a valid loop latch"); + + const BranchInst *BI = dyn_cast<BranchInst>(Latch->getTerminator()); + assert(BI && BI->isConditional() && + "Expecting loop latch terminator to be a branch instruction"); + + CmpInst *OuterLoopLatchCmp = dyn_cast<CmpInst>(BI->getCondition()); + DEBUG_WITH_TYPE( + VerboseDebug, if (OuterLoopLatchCmp) { + dbgs() << "Outer loop latch compare instruction: " << *OuterLoopLatchCmp + << "\n"; + }); + return OuterLoopLatchCmp; +} + +static CmpInst *getInnerLoopGuardCmp(const Loop &InnerLoop) { + + BranchInst *InnerGuard = InnerLoop.getLoopGuardBranch(); + CmpInst *InnerLoopGuardCmp = + (InnerGuard) ? dyn_cast<CmpInst>(InnerGuard->getCondition()) : nullptr; + + DEBUG_WITH_TYPE( + VerboseDebug, if (InnerLoopGuardCmp) { + dbgs() << "Inner loop guard compare instruction: " << *InnerLoopGuardCmp + << "\n"; + }); + return InnerLoopGuardCmp; +} + +static bool checkSafeInstruction(const Instruction &I, + const CmpInst *InnerLoopGuardCmp, + const CmpInst *OuterLoopLatchCmp, + Optional<Loop::LoopBounds> OuterLoopLB) { + + bool IsAllowed = + isSafeToSpeculativelyExecute(&I) || isa<PHINode>(I) || isa<BranchInst>(I); + if (!IsAllowed) + return false; + // The only binary instruction allowed is the outer loop step instruction, + // the only comparison instructions allowed are the inner loop guard + // compare instruction and the outer loop latch compare instruction. + if ((isa<BinaryOperator>(I) && &I != &OuterLoopLB->getStepInst()) || + (isa<CmpInst>(I) && &I != OuterLoopLatchCmp && &I != InnerLoopGuardCmp)) { + return false; + } + return true; +} + bool LoopNest::arePerfectlyNested(const Loop &OuterLoop, const Loop &InnerLoop, ScalarEvolution &SE) { + return (analyzeLoopNestForPerfectNest(OuterLoop, InnerLoop, SE) == + PerfectLoopNest); +} + +LoopNest::LoopNestEnum LoopNest::analyzeLoopNestForPerfectNest( + const Loop &OuterLoop, const Loop &InnerLoop, ScalarEvolution &SE) { + assert(!OuterLoop.isInnermost() && "Outer loop should have subloops"); assert(!InnerLoop.isOutermost() && "Inner loop should have a parent"); LLVM_DEBUG(dbgs() << "Checking whether loop '" << OuterLoop.getName() @@ -66,7 +124,7 @@ bool LoopNest::arePerfectlyNested(const Loop &OuterLoop, const Loop &InnerLoop, // the outer loop latch. if (!checkLoopsStructure(OuterLoop, InnerLoop, SE)) { LLVM_DEBUG(dbgs() << "Not perfectly nested: invalid loop structure.\n"); - return false; + return InvalidLoopStructure; } // Bail out if we cannot retrieve the outer loop bounds. @@ -74,33 +132,11 @@ bool LoopNest::arePerfectlyNested(const Loop &OuterLoop, const Loop &InnerLoop, if (OuterLoopLB == None) { LLVM_DEBUG(dbgs() << "Cannot compute loop bounds of OuterLoop: " << OuterLoop << "\n";); - return false; + return OuterLoopLowerBoundUnknown; } - // Identify the outer loop latch comparison instruction. - const BasicBlock *Latch = OuterLoop.getLoopLatch(); - assert(Latch && "Expecting a valid loop latch"); - const BranchInst *BI = dyn_cast<BranchInst>(Latch->getTerminator()); - assert(BI && BI->isConditional() && - "Expecting loop latch terminator to be a branch instruction"); - - const CmpInst *OuterLoopLatchCmp = dyn_cast<CmpInst>(BI->getCondition()); - DEBUG_WITH_TYPE( - VerboseDebug, if (OuterLoopLatchCmp) { - dbgs() << "Outer loop latch compare instruction: " << *OuterLoopLatchCmp - << "\n"; - }); - - // Identify the inner loop guard instruction. - BranchInst *InnerGuard = InnerLoop.getLoopGuardBranch(); - const CmpInst *InnerLoopGuardCmp = - (InnerGuard) ? dyn_cast<CmpInst>(InnerGuard->getCondition()) : nullptr; - - DEBUG_WITH_TYPE( - VerboseDebug, if (InnerLoopGuardCmp) { - dbgs() << "Inner loop guard compare instruction: " << *InnerLoopGuardCmp - << "\n"; - }); + CmpInst *OuterLoopLatchCmp = getOuterLoopLatchCmp(OuterLoop); + CmpInst *InnerLoopGuardCmp = getInnerLoopGuardCmp(InnerLoop); // Determine whether instructions in a basic block are one of: // - the inner loop guard comparison @@ -109,29 +145,15 @@ bool LoopNest::arePerfectlyNested(const Loop &OuterLoop, const Loop &InnerLoop, // - a phi node, a cast or a branch auto containsOnlySafeInstructions = [&](const BasicBlock &BB) { return llvm::all_of(BB, [&](const Instruction &I) { - bool isAllowed = isSafeToSpeculativelyExecute(&I) || isa<PHINode>(I) || - isa<BranchInst>(I); - if (!isAllowed) { - DEBUG_WITH_TYPE(VerboseDebug, { - dbgs() << "Instruction: " << I << "\nin basic block: " << BB - << " is considered unsafe.\n"; - }); - return false; - } - - // The only binary instruction allowed is the outer loop step instruction, - // the only comparison instructions allowed are the inner loop guard - // compare instruction and the outer loop latch compare instruction. - if ((isa<BinaryOperator>(I) && &I != &OuterLoopLB->getStepInst()) || - (isa<CmpInst>(I) && &I != OuterLoopLatchCmp && - &I != InnerLoopGuardCmp)) { + bool IsSafeInstr = checkSafeInstruction(I, InnerLoopGuardCmp, + OuterLoopLatchCmp, OuterLoopLB); + if (IsSafeInstr) { DEBUG_WITH_TYPE(VerboseDebug, { dbgs() << "Instruction: " << I << "\nin basic block:" << BB << "is unsafe.\n"; }); - return false; } - return true; + return IsSafeInstr; }); }; @@ -148,13 +170,72 @@ bool LoopNest::arePerfectlyNested(const Loop &OuterLoop, const Loop &InnerLoop, !containsOnlySafeInstructions(*InnerLoop.getExitBlock())) { LLVM_DEBUG(dbgs() << "Not perfectly nested: code surrounding inner loop is " "unsafe\n";); - return false; + return ImperfectLoopNest; } LLVM_DEBUG(dbgs() << "Loop '" << OuterLoop.getName() << "' and '" << InnerLoop.getName() << "' are perfectly nested.\n"); - return true; + return PerfectLoopNest; +} + +LoopNest::InstrVectorTy LoopNest::getInterveningInstructions( + const Loop &OuterLoop, const Loop &InnerLoop, ScalarEvolution &SE) { + InstrVectorTy Instr; + switch (analyzeLoopNestForPerfectNest(OuterLoop, InnerLoop, SE)) { + case PerfectLoopNest: + LLVM_DEBUG(dbgs() << "The loop Nest is Perfect, returning empty " + "instruction vector. \n";); + return Instr; + + case InvalidLoopStructure: + LLVM_DEBUG(dbgs() << "Not perfectly nested: invalid loop structure. " + "Instruction vector is empty.\n";); + return Instr; + + case OuterLoopLowerBoundUnknown: + LLVM_DEBUG(dbgs() << "Cannot compute loop bounds of OuterLoop: " + << OuterLoop << "\nInstruction vector is empty.\n";); + return Instr; + + case ImperfectLoopNest: + break; + } + + // Identify the outer loop latch comparison instruction. + auto OuterLoopLB = OuterLoop.getBounds(SE); + + CmpInst *OuterLoopLatchCmp = getOuterLoopLatchCmp(OuterLoop); + CmpInst *InnerLoopGuardCmp = getInnerLoopGuardCmp(InnerLoop); + + auto GetUnsafeInstructions = [&](const BasicBlock &BB) { + for (const Instruction &I : BB) { + if (!checkSafeInstruction(I, InnerLoopGuardCmp, OuterLoopLatchCmp, + OuterLoopLB)) { + Instr.push_back(&I); + DEBUG_WITH_TYPE(VerboseDebug, { + dbgs() << "Instruction: " << I << "\nin basic block:" << BB + << "is unsafe.\n"; + }); + } + } + }; + + // Check the code surrounding the inner loop for instructions that are deemed + // unsafe. + const BasicBlock *OuterLoopHeader = OuterLoop.getHeader(); + const BasicBlock *OuterLoopLatch = OuterLoop.getLoopLatch(); + const BasicBlock *InnerLoopPreHeader = InnerLoop.getLoopPreheader(); + const BasicBlock *InnerLoopExitBlock = InnerLoop.getExitBlock(); + + GetUnsafeInstructions(*OuterLoopHeader); + GetUnsafeInstructions(*OuterLoopLatch); + GetUnsafeInstructions(*InnerLoopExitBlock); + + if (InnerLoopPreHeader != OuterLoopHeader) { + GetUnsafeInstructions(*InnerLoopPreHeader); + } + return Instr; } SmallVector<LoopVectorTy, 4> diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp index 5b95ed223fd9..6fc4c42bdd71 100644 --- a/llvm/lib/Analysis/MLInlineAdvisor.cpp +++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp @@ -116,6 +116,8 @@ MLInlineAdvisor::MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, void MLInlineAdvisor::onPassEntry() { // Function passes executed between InlinerPass runs may have changed the // module-wide features. + if (!Invalid) + return; NodeCount = 0; EdgeCount = 0; for (auto &F : M) @@ -123,6 +125,7 @@ void MLInlineAdvisor::onPassEntry() { ++NodeCount; EdgeCount += getLocalCalls(F); } + Invalid = false; } int64_t MLInlineAdvisor::getLocalCalls(Function &F) { diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp index 68e997656d84..4f2b5b34304d 100644 --- a/llvm/lib/Analysis/MemoryBuiltins.cpp +++ b/llvm/lib/Analysis/MemoryBuiltins.cpp @@ -111,7 +111,7 @@ static const std::pair<LibFunc, AllocFnsTy> AllocationFnData[] = { {LibFunc_reallocf, {ReallocLike, 2, 1, -1}}, {LibFunc_strdup, {StrDupLike, 1, -1, -1}}, {LibFunc_strndup, {StrDupLike, 2, 1, -1}}, - {LibFunc___kmpc_alloc_shared, {MallocLike, 1, 0, -1}} + {LibFunc___kmpc_alloc_shared, {MallocLike, 1, 0, -1}}, // TODO: Handle "int posix_memalign(void **, size_t, size_t)" }; @@ -135,9 +135,8 @@ static const Function *getCalledFunction(const Value *V, bool LookThroughBitCast return nullptr; } -/// Returns the allocation data for the given value if it's either a call to a -/// known allocation function, or a call to a function with the allocsize -/// attribute. +/// Returns the allocation data for the given value if it's a call to a known +/// allocation function. static Optional<AllocFnsTy> getAllocationDataForFunction(const Function *Callee, AllocType AllocTy, const TargetLibraryInfo *TLI) { @@ -610,7 +609,7 @@ ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const DataLayout &DL, SizeOffsetType ObjectSizeOffsetVisitor::compute(Value *V) { IntTyBits = DL.getIndexTypeSizeInBits(V->getType()); - Zero = APInt::getNullValue(IntTyBits); + Zero = APInt::getZero(IntTyBits); V = V->stripPointerCasts(); if (Instruction *I = dyn_cast<Instruction>(V)) { diff --git a/llvm/lib/Analysis/MemoryLocation.cpp b/llvm/lib/Analysis/MemoryLocation.cpp index ef9cda37ce35..7f2d04c49565 100644 --- a/llvm/lib/Analysis/MemoryLocation.cpp +++ b/llvm/lib/Analysis/MemoryLocation.cpp @@ -35,54 +35,44 @@ void LocationSize::print(raw_ostream &OS) const { } MemoryLocation MemoryLocation::get(const LoadInst *LI) { - AAMDNodes AATags; - LI->getAAMetadata(AATags); const auto &DL = LI->getModule()->getDataLayout(); return MemoryLocation( LI->getPointerOperand(), - LocationSize::precise(DL.getTypeStoreSize(LI->getType())), AATags); + LocationSize::precise(DL.getTypeStoreSize(LI->getType())), + LI->getAAMetadata()); } MemoryLocation MemoryLocation::get(const StoreInst *SI) { - AAMDNodes AATags; - SI->getAAMetadata(AATags); const auto &DL = SI->getModule()->getDataLayout(); return MemoryLocation(SI->getPointerOperand(), LocationSize::precise(DL.getTypeStoreSize( SI->getValueOperand()->getType())), - AATags); + SI->getAAMetadata()); } MemoryLocation MemoryLocation::get(const VAArgInst *VI) { - AAMDNodes AATags; - VI->getAAMetadata(AATags); - return MemoryLocation(VI->getPointerOperand(), - LocationSize::afterPointer(), AATags); + LocationSize::afterPointer(), VI->getAAMetadata()); } MemoryLocation MemoryLocation::get(const AtomicCmpXchgInst *CXI) { - AAMDNodes AATags; - CXI->getAAMetadata(AATags); const auto &DL = CXI->getModule()->getDataLayout(); return MemoryLocation(CXI->getPointerOperand(), LocationSize::precise(DL.getTypeStoreSize( CXI->getCompareOperand()->getType())), - AATags); + CXI->getAAMetadata()); } MemoryLocation MemoryLocation::get(const AtomicRMWInst *RMWI) { - AAMDNodes AATags; - RMWI->getAAMetadata(AATags); const auto &DL = RMWI->getModule()->getDataLayout(); return MemoryLocation(RMWI->getPointerOperand(), LocationSize::precise(DL.getTypeStoreSize( RMWI->getValOperand()->getType())), - AATags); + RMWI->getAAMetadata()); } Optional<MemoryLocation> MemoryLocation::getOrNone(const Instruction *Inst) { @@ -117,10 +107,7 @@ MemoryLocation MemoryLocation::getForSource(const AnyMemTransferInst *MTI) { // memcpy/memmove can have AA tags. For memcpy, they apply // to both the source and the destination. - AAMDNodes AATags; - MTI->getAAMetadata(AATags); - - return MemoryLocation(MTI->getRawSource(), Size, AATags); + return MemoryLocation(MTI->getRawSource(), Size, MTI->getAAMetadata()); } MemoryLocation MemoryLocation::getForDest(const MemIntrinsic *MI) { @@ -138,17 +125,13 @@ MemoryLocation MemoryLocation::getForDest(const AnyMemIntrinsic *MI) { // memcpy/memmove can have AA tags. For memcpy, they apply // to both the source and the destination. - AAMDNodes AATags; - MI->getAAMetadata(AATags); - - return MemoryLocation(MI->getRawDest(), Size, AATags); + return MemoryLocation(MI->getRawDest(), Size, MI->getAAMetadata()); } MemoryLocation MemoryLocation::getForArgument(const CallBase *Call, unsigned ArgIdx, const TargetLibraryInfo *TLI) { - AAMDNodes AATags; - Call->getAAMetadata(AATags); + AAMDNodes AATags = Call->getAAMetadata(); const Value *Arg = Call->getArgOperand(ArgIdx); // We may be able to produce an exact size for known intrinsics. diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp index b402b0467f5d..ac20e20f0c0d 100644 --- a/llvm/lib/Analysis/MemorySSA.cpp +++ b/llvm/lib/Analysis/MemorySSA.cpp @@ -90,22 +90,18 @@ bool llvm::VerifyMemorySSA = true; #else bool llvm::VerifyMemorySSA = false; #endif -/// Enables memory ssa as a dependency for loop passes in legacy pass manager. -cl::opt<bool> llvm::EnableMSSALoopDependency( - "enable-mssa-loop-dependency", cl::Hidden, cl::init(true), - cl::desc("Enable MemorySSA dependency for loop pass manager")); static cl::opt<bool, true> VerifyMemorySSAX("verify-memoryssa", cl::location(VerifyMemorySSA), cl::Hidden, cl::desc("Enable verification of MemorySSA.")); -namespace llvm { +const static char LiveOnEntryStr[] = "liveOnEntry"; + +namespace { /// An assembly annotator class to print Memory SSA information in /// comments. class MemorySSAAnnotatedWriter : public AssemblyAnnotationWriter { - friend class MemorySSA; - const MemorySSA *MSSA; public: @@ -124,7 +120,34 @@ public: } }; -} // end namespace llvm +/// An assembly annotator class to print Memory SSA information in +/// comments. +class MemorySSAWalkerAnnotatedWriter : public AssemblyAnnotationWriter { + MemorySSA *MSSA; + MemorySSAWalker *Walker; + +public: + MemorySSAWalkerAnnotatedWriter(MemorySSA *M) + : MSSA(M), Walker(M->getWalker()) {} + + void emitInstructionAnnot(const Instruction *I, + formatted_raw_ostream &OS) override { + if (MemoryAccess *MA = MSSA->getMemoryAccess(I)) { + MemoryAccess *Clobber = Walker->getClobberingMemoryAccess(MA); + OS << "; " << *MA; + if (Clobber) { + OS << " - clobbered by "; + if (MSSA->isLiveOnEntryDef(Clobber)) + OS << LiveOnEntryStr; + else + OS << *Clobber; + } + OS << "\n"; + } + } +}; + +} // namespace namespace { @@ -286,6 +309,7 @@ instructionClobbersQuery(const MemoryDef *MD, const MemoryLocation &UseLoc, case Intrinsic::invariant_end: case Intrinsic::assume: case Intrinsic::experimental_noalias_scope_decl: + case Intrinsic::pseudoprobe: return {false, AliasResult(AliasResult::NoAlias)}; case Intrinsic::dbg_addr: case Intrinsic::dbg_declare: @@ -1016,7 +1040,8 @@ public: // updated if a new clobber is found by this SkipSelf search. If this // additional query becomes heavily used we may decide to cache the result. // Walker instantiations will decide how to set the SkipSelf bool. - MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *, unsigned &, bool); + MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *, unsigned &, bool, + bool UseInvariantGroup = true); }; /// A MemorySSAWalker that does AA walks to disambiguate accesses. It no @@ -1041,6 +1066,11 @@ public: unsigned &UWL) { return Walker->getClobberingMemoryAccessBase(MA, Loc, UWL); } + // This method is not accessible outside of this file. + MemoryAccess *getClobberingMemoryAccessWithoutInvariantGroup(MemoryAccess *MA, + unsigned &UWL) { + return Walker->getClobberingMemoryAccessBase(MA, UWL, false, false); + } MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override { unsigned UpwardWalkLimit = MaxCheckLimit; @@ -1437,10 +1467,13 @@ void MemorySSA::OptimizeUses::optimizeUsesInBlock( unsigned UpwardWalkLimit = MaxCheckLimit; while (UpperBound > LocInfo.LowerBound) { if (isa<MemoryPhi>(VersionStack[UpperBound])) { - // For phis, use the walker, see where we ended up, go there + // For phis, use the walker, see where we ended up, go there. + // The invariant.group handling in MemorySSA is ad-hoc and doesn't + // support updates, so don't use it to optimize uses. MemoryAccess *Result = - Walker->getClobberingMemoryAccess(MU, UpwardWalkLimit); - // We are guaranteed to find it or something is wrong + Walker->getClobberingMemoryAccessWithoutInvariantGroup( + MU, UpwardWalkLimit); + // We are guaranteed to find it or something is wrong. while (VersionStack[UpperBound] != Result) { assert(UpperBound != 0); --UpperBound; @@ -1750,6 +1783,7 @@ MemoryUseOrDef *MemorySSA::createNewAccess(Instruction *I, break; case Intrinsic::assume: case Intrinsic::experimental_noalias_scope_decl: + case Intrinsic::pseudoprobe: return nullptr; } } @@ -1864,10 +1898,17 @@ void MemorySSA::print(raw_ostream &OS) const { LLVM_DUMP_METHOD void MemorySSA::dump() const { print(dbgs()); } #endif -void MemorySSA::verifyMemorySSA() const { - verifyOrderingDominationAndDefUses(F); +void MemorySSA::verifyMemorySSA(VerificationLevel VL) const { +#if !defined(NDEBUG) && defined(EXPENSIVE_CHECKS) + VL = VerificationLevel::Full; +#endif + +#ifndef NDEBUG + verifyOrderingDominationAndDefUses(F, VL); verifyDominationNumbers(F); - verifyPrevDefInPhis(F); + if (VL == VerificationLevel::Full) + verifyPrevDefInPhis(F); +#endif // Previously, the verification used to also verify that the clobberingAccess // cached by MemorySSA is the same as the clobberingAccess found at a later // query to AA. This does not hold true in general due to the current fragility @@ -1881,7 +1922,6 @@ void MemorySSA::verifyMemorySSA() const { } void MemorySSA::verifyPrevDefInPhis(Function &F) const { -#if !defined(NDEBUG) && defined(EXPENSIVE_CHECKS) for (const BasicBlock &BB : F) { if (MemoryPhi *Phi = getMemoryAccess(&BB)) { for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) { @@ -1896,6 +1936,8 @@ void MemorySSA::verifyPrevDefInPhis(Function &F) const { auto *LastAcc = &*(--DefList->end()); assert(LastAcc == IncAcc && "Incorrect incoming access into phi."); + (void)IncAcc; + (void)LastAcc; break; } DTNode = DTNode->getIDom(); @@ -1911,13 +1953,11 @@ void MemorySSA::verifyPrevDefInPhis(Function &F) const { } } } -#endif } /// Verify that all of the blocks we believe to have valid domination numbers /// actually have valid domination numbers. void MemorySSA::verifyDominationNumbers(const Function &F) const { -#ifndef NDEBUG if (BlockNumberingValid.empty()) return; @@ -1943,13 +1983,13 @@ void MemorySSA::verifyDominationNumbers(const Function &F) const { unsigned long ThisNumber = ThisNumberIter->second; assert(ThisNumber > LastNumber && "Domination numbers should be strictly increasing!"); + (void)LastNumber; LastNumber = ThisNumber; } } assert(ValidBlocks.empty() && "All valid BasicBlocks should exist in F -- dangling pointers?"); -#endif } /// Verify ordering: the order and existence of MemoryAccesses matches the @@ -1958,8 +1998,8 @@ void MemorySSA::verifyDominationNumbers(const Function &F) const { /// Verify def-uses: the immediate use information - walk all the memory /// accesses and verifying that, for each use, it appears in the appropriate /// def's use list -void MemorySSA::verifyOrderingDominationAndDefUses(Function &F) const { -#if !defined(NDEBUG) +void MemorySSA::verifyOrderingDominationAndDefUses(Function &F, + VerificationLevel VL) const { // Walk all the blocks, comparing what the lookups think and what the access // lists think, as well as the order in the blocks vs the order in the access // lists. @@ -1974,19 +2014,21 @@ void MemorySSA::verifyOrderingDominationAndDefUses(Function &F) const { ActualAccesses.push_back(Phi); ActualDefs.push_back(Phi); // Verify domination - for (const Use &U : Phi->uses()) + for (const Use &U : Phi->uses()) { assert(dominates(Phi, U) && "Memory PHI does not dominate it's uses"); -#if defined(EXPENSIVE_CHECKS) - // Verify def-uses. - assert(Phi->getNumOperands() == static_cast<unsigned>(std::distance( - pred_begin(&B), pred_end(&B))) && - "Incomplete MemoryPhi Node"); - for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) { - verifyUseInDefs(Phi->getIncomingValue(I), Phi); - assert(is_contained(predecessors(&B), Phi->getIncomingBlock(I)) && - "Incoming phi block not a block predecessor"); + (void)U; + } + // Verify def-uses for full verify. + if (VL == VerificationLevel::Full) { + assert(Phi->getNumOperands() == static_cast<unsigned>(std::distance( + pred_begin(&B), pred_end(&B))) && + "Incomplete MemoryPhi Node"); + for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) { + verifyUseInDefs(Phi->getIncomingValue(I), Phi); + assert(is_contained(predecessors(&B), Phi->getIncomingBlock(I)) && + "Incoming phi block not a block predecessor"); + } } -#endif } for (Instruction &I : B) { @@ -2002,14 +2044,15 @@ void MemorySSA::verifyOrderingDominationAndDefUses(Function &F) const { // Verify ordering. ActualDefs.push_back(MA); // Verify domination. - for (const Use &U : MD->uses()) + for (const Use &U : MD->uses()) { assert(dominates(MD, U) && "Memory Def does not dominate it's uses"); + (void)U; + } } -#if defined(EXPENSIVE_CHECKS) - // Verify def-uses. - verifyUseInDefs(MA->getDefiningAccess(), MA); -#endif + // Verify def-uses for full verify. + if (VL == VerificationLevel::Full) + verifyUseInDefs(MA->getDefiningAccess(), MA); } } // Either we hit the assert, really have no accesses, or we have both @@ -2044,13 +2087,11 @@ void MemorySSA::verifyOrderingDominationAndDefUses(Function &F) const { } ActualDefs.clear(); } -#endif } /// Verify the def-use lists in MemorySSA, by verifying that \p Use /// appears in the use list of \p Def. void MemorySSA::verifyUseInDefs(MemoryAccess *Def, MemoryAccess *Use) const { -#ifndef NDEBUG // The live on entry use may cause us to get a NULL def here if (!Def) assert(isLiveOnEntryDef(Use) && @@ -2058,7 +2099,6 @@ void MemorySSA::verifyUseInDefs(MemoryAccess *Def, MemoryAccess *Use) const { else assert(is_contained(Def->users(), Use) && "Did not find use in def's use list"); -#endif } /// Perform a local numbering on blocks so that instruction ordering can be @@ -2138,8 +2178,6 @@ bool MemorySSA::dominates(const MemoryAccess *Dominator, return dominates(Dominator, cast<MemoryAccess>(Dominatee.getUser())); } -const static char LiveOnEntryStr[] = "liveOnEntry"; - void MemoryAccess::print(raw_ostream &OS) const { switch (getValueID()) { case MemoryPhiVal: return static_cast<const MemoryPhi *>(this)->print(OS); @@ -2355,6 +2393,16 @@ PreservedAnalyses MemorySSAPrinterPass::run(Function &F, return PreservedAnalyses::all(); } +PreservedAnalyses MemorySSAWalkerPrinterPass::run(Function &F, + FunctionAnalysisManager &AM) { + auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA(); + OS << "MemorySSA (walker) for function: " << F.getName() << "\n"; + MemorySSAWalkerAnnotatedWriter Writer(&MSSA); + F.print(OS, &Writer); + + return PreservedAnalyses::all(); +} + PreservedAnalyses MemorySSAVerifierPass::run(Function &F, FunctionAnalysisManager &AM) { AM.getResult<MemorySSAAnalysis>(F).getMSSA().verifyMemorySSA(); @@ -2438,15 +2486,88 @@ MemorySSA::ClobberWalkerBase<AliasAnalysisType>::getClobberingMemoryAccessBase( return Clobber; } +static const Instruction * +getInvariantGroupClobberingInstruction(Instruction &I, DominatorTree &DT) { + if (!I.hasMetadata(LLVMContext::MD_invariant_group) || I.isVolatile()) + return nullptr; + + // We consider bitcasts and zero GEPs to be the same pointer value. Start by + // stripping bitcasts and zero GEPs, then we will recursively look at loads + // and stores through bitcasts and zero GEPs. + Value *PointerOperand = getLoadStorePointerOperand(&I)->stripPointerCasts(); + + // It's not safe to walk the use list of a global value because function + // passes aren't allowed to look outside their functions. + // FIXME: this could be fixed by filtering instructions from outside of + // current function. + if (isa<Constant>(PointerOperand)) + return nullptr; + + // Queue to process all pointers that are equivalent to load operand. + SmallVector<const Value *, 8> PointerUsesQueue; + PointerUsesQueue.push_back(PointerOperand); + + const Instruction *MostDominatingInstruction = &I; + + // FIXME: This loop is O(n^2) because dominates can be O(n) and in worst case + // we will see all the instructions. It may not matter in practice. If it + // does, we will have to support MemorySSA construction and updates. + while (!PointerUsesQueue.empty()) { + const Value *Ptr = PointerUsesQueue.pop_back_val(); + assert(Ptr && !isa<GlobalValue>(Ptr) && + "Null or GlobalValue should not be inserted"); + + for (const User *Us : Ptr->users()) { + auto *U = dyn_cast<Instruction>(Us); + if (!U || U == &I || !DT.dominates(U, MostDominatingInstruction)) + continue; + + // Add bitcasts and zero GEPs to queue. + if (isa<BitCastInst>(U)) { + PointerUsesQueue.push_back(U); + continue; + } + if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) { + if (GEP->hasAllZeroIndices()) + PointerUsesQueue.push_back(U); + continue; + } + + // If we hit a load/store with an invariant.group metadata and the same + // pointer operand, we can assume that value pointed to by the pointer + // operand didn't change. + if (U->hasMetadata(LLVMContext::MD_invariant_group) && + getLoadStorePointerOperand(U) == Ptr && !U->isVolatile()) { + MostDominatingInstruction = U; + } + } + } + return MostDominatingInstruction == &I ? nullptr : MostDominatingInstruction; +} + template <typename AliasAnalysisType> MemoryAccess * MemorySSA::ClobberWalkerBase<AliasAnalysisType>::getClobberingMemoryAccessBase( - MemoryAccess *MA, unsigned &UpwardWalkLimit, bool SkipSelf) { + MemoryAccess *MA, unsigned &UpwardWalkLimit, bool SkipSelf, + bool UseInvariantGroup) { auto *StartingAccess = dyn_cast<MemoryUseOrDef>(MA); // If this is a MemoryPhi, we can't do anything. if (!StartingAccess) return MA; + if (UseInvariantGroup) { + if (auto *I = getInvariantGroupClobberingInstruction( + *StartingAccess->getMemoryInst(), MSSA->getDomTree())) { + assert(isa<LoadInst>(I) || isa<StoreInst>(I)); + + auto *ClobberMA = MSSA->getMemoryAccess(I); + assert(ClobberMA); + if (isa<MemoryUse>(ClobberMA)) + return ClobberMA->getDefiningAccess(); + return ClobberMA; + } + } + bool IsOptimized = false; // If this is an already optimized use or def, return the optimized result. diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp index 616864f360bf..9c841883de6d 100644 --- a/llvm/lib/Analysis/MemorySSAUpdater.cpp +++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp @@ -296,9 +296,8 @@ static void setMemoryPhiValueForBlock(MemoryPhi *MP, const BasicBlock *BB, assert(i != -1 && "Should have found the basic block in the phi"); // We can't just compare i against getNumOperands since one is signed and the // other not. So use it to index into the block iterator. - for (auto BBIter = MP->block_begin() + i; BBIter != MP->block_end(); - ++BBIter) { - if (*BBIter != BB) + for (const BasicBlock *BlockBB : llvm::drop_begin(MP->blocks(), i)) { + if (BlockBB != BB) break; MP->setIncomingValue(i, NewDef); ++i; @@ -491,8 +490,7 @@ void MemorySSAUpdater::fixupDefs(const SmallVectorImpl<WeakVH> &Vars) { } while (!Worklist.empty()) { - const BasicBlock *FixupBlock = Worklist.back(); - Worklist.pop_back(); + const BasicBlock *FixupBlock = Worklist.pop_back_val(); // Get the first def in the block that isn't a phi node. if (auto *Defs = MSSA->getWritableBlockDefs(FixupBlock)) { @@ -822,25 +820,30 @@ void MemorySSAUpdater::applyUpdates(ArrayRef<CFGUpdate> Updates, } if (!DeleteUpdates.empty()) { - if (!UpdateDT) { - SmallVector<CFGUpdate, 0> Empty; - // Deletes are reversed applied, because this CFGView is pretending the - // deletes did not happen yet, hence the edges still exist. - DT.applyUpdates(Empty, RevDeleteUpdates); + if (!InsertUpdates.empty()) { + if (!UpdateDT) { + SmallVector<CFGUpdate, 0> Empty; + // Deletes are reversed applied, because this CFGView is pretending the + // deletes did not happen yet, hence the edges still exist. + DT.applyUpdates(Empty, RevDeleteUpdates); + } else { + // Apply all updates, with the RevDeleteUpdates as PostCFGView. + DT.applyUpdates(Updates, RevDeleteUpdates); + } + + // Note: the MSSA update below doesn't distinguish between a GD with + // (RevDelete,false) and (Delete, true), but this matters for the DT + // updates above; for "children" purposes they are equivalent; but the + // updates themselves convey the desired update, used inside DT only. + GraphDiff<BasicBlock *> GD(RevDeleteUpdates); + applyInsertUpdates(InsertUpdates, DT, &GD); + // Update DT to redelete edges; this matches the real CFG so we can + // perform the standard update without a postview of the CFG. + DT.applyUpdates(DeleteUpdates); } else { - // Apply all updates, with the RevDeleteUpdates as PostCFGView. - DT.applyUpdates(Updates, RevDeleteUpdates); + if (UpdateDT) + DT.applyUpdates(DeleteUpdates); } - - // Note: the MSSA update below doesn't distinguish between a GD with - // (RevDelete,false) and (Delete, true), but this matters for the DT - // updates above; for "children" purposes they are equivalent; but the - // updates themselves convey the desired update, used inside DT only. - GraphDiff<BasicBlock *> GD(RevDeleteUpdates); - applyInsertUpdates(InsertUpdates, DT, &GD); - // Update DT to redelete edges; this matches the real CFG so we can perform - // the standard update without a postview of the CFG. - DT.applyUpdates(DeleteUpdates); } else { if (UpdateDT) DT.applyUpdates(Updates); @@ -1131,11 +1134,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates, if (auto DefsList = MSSA->getWritableBlockDefs(BlockWithDefsToReplace)) { for (auto &DefToReplaceUses : *DefsList) { BasicBlock *DominatingBlock = DefToReplaceUses.getBlock(); - Value::use_iterator UI = DefToReplaceUses.use_begin(), - E = DefToReplaceUses.use_end(); - for (; UI != E;) { - Use &U = *UI; - ++UI; + for (Use &U : llvm::make_early_inc_range(DefToReplaceUses.uses())) { MemoryAccess *Usr = cast<MemoryAccess>(U.getUser()); if (MemoryPhi *UsrPhi = dyn_cast<MemoryPhi>(Usr)) { BasicBlock *DominatedBlock = UsrPhi->getIncomingBlock(U); diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index e43553222128..d80814852e19 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -264,11 +264,27 @@ static void computeFunctionSummary( std::vector<const Instruction *> NonVolatileStores; bool HasInlineAsmMaybeReferencingInternal = false; - for (const BasicBlock &BB : F) + bool HasIndirBranchToBlockAddress = false; + bool HasUnknownCall = false; + bool MayThrow = false; + for (const BasicBlock &BB : F) { + // We don't allow inlining of function with indirect branch to blockaddress. + // If the blockaddress escapes the function, e.g., via a global variable, + // inlining may lead to an invalid cross-function reference. So we shouldn't + // import such function either. + if (BB.hasAddressTaken()) { + for (User *U : BlockAddress::get(const_cast<BasicBlock *>(&BB))->users()) + if (!isa<CallBrInst>(*U)) { + HasIndirBranchToBlockAddress = true; + break; + } + } + for (const Instruction &I : BB) { - if (isa<DbgInfoIntrinsic>(I)) + if (I.isDebugOrPseudoInst()) continue; ++NumInsts; + // Regular LTO module doesn't participate in ThinLTO import, // so no reference from it can be read/writeonly, since this // would require importing variable as local copy @@ -300,8 +316,11 @@ static void computeFunctionSummary( } findRefEdges(Index, &I, RefEdges, Visited); const auto *CB = dyn_cast<CallBase>(&I); - if (!CB) + if (!CB) { + if (I.mayThrow()) + MayThrow = true; continue; + } const auto *CI = dyn_cast<CallInst>(&I); // Since we don't know exactly which local values are referenced in inline @@ -323,7 +342,7 @@ static void computeFunctionSummary( // called aliasee for the checks below. if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) { assert(!CalledFunction && "Expected null called function in callsite for alias"); - CalledFunction = dyn_cast<Function>(GA->getBaseObject()); + CalledFunction = dyn_cast<Function>(GA->getAliaseeObject()); } // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. @@ -357,6 +376,7 @@ static void computeFunctionSummary( ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq); } } else { + HasUnknownCall = true; // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; @@ -386,6 +406,7 @@ static void computeFunctionSummary( .updateHotness(getHotness(Candidate.Count, PSI)); } } + } Index.addBlockCount(F.size()); std::vector<ValueInfo> Refs; @@ -452,8 +473,9 @@ static void computeFunctionSummary( : CalleeInfo::HotnessType::Critical); bool NonRenamableLocal = isNonRenamableLocal(F); - bool NotEligibleForImport = - NonRenamableLocal || HasInlineAsmMaybeReferencingInternal; + bool NotEligibleForImport = NonRenamableLocal || + HasInlineAsmMaybeReferencingInternal || + HasIndirBranchToBlockAddress; GlobalValueSummary::GVFlags Flags( F.getLinkage(), F.getVisibility(), NotEligibleForImport, /* Live = */ false, F.isDSOLocal(), @@ -464,8 +486,9 @@ static void computeFunctionSummary( F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(), // FIXME: refactor this to use the same code that inliner is using. // Don't try to import functions with noinline attribute. - F.getAttributes().hasFnAttribute(Attribute::NoInline), - F.hasFnAttribute(Attribute::AlwaysInline)}; + F.getAttributes().hasFnAttr(Attribute::NoInline), + F.hasFnAttribute(Attribute::AlwaysInline), + F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall}; std::vector<FunctionSummary::ParamAccess> ParamAccesses; if (auto *SSI = GetSSICallback(F)) ParamAccesses = SSI->getParamAccesses(Index); @@ -622,7 +645,7 @@ computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, /* Live = */ false, A.isDSOLocal(), A.hasLinkOnceODRLinkage() && A.hasGlobalUnnamedAddr()); auto AS = std::make_unique<AliasSummary>(Flags); - auto *Aliasee = A.getBaseObject(); + auto *Aliasee = A.getAliaseeObject(); auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID()); assert(AliaseeVI && "Alias expects aliasee summary to be available"); assert(AliaseeVI.getSummaryList().size() == 1 && @@ -711,7 +734,10 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( F->hasFnAttribute(Attribute::NoRecurse), F->returnDoesNotAlias(), /* NoInline = */ false, - F->hasFnAttribute(Attribute::AlwaysInline)}, + F->hasFnAttribute(Attribute::AlwaysInline), + F->hasFnAttribute(Attribute::NoUnwind), + /* MayThrow */ true, + /* HasUnknownCall */ true}, /*EntryCount=*/0, ArrayRef<ValueInfo>{}, ArrayRef<FunctionSummary::EdgeTy>{}, ArrayRef<GlobalValue::GUID>{}, diff --git a/llvm/lib/Analysis/ObjCARCInstKind.cpp b/llvm/lib/Analysis/ObjCARCInstKind.cpp index 704d15f3280d..f74a9f7f104f 100644 --- a/llvm/lib/Analysis/ObjCARCInstKind.cpp +++ b/llvm/lib/Analysis/ObjCARCInstKind.cpp @@ -296,9 +296,8 @@ ARCInstKind llvm::objcarc::GetARCInstKind(const Value *V) { // operand isn't actually being dereferenced, it is being stored to // memory where we can no longer track who might read it and dereference // it, so we have to consider it potentially used. - for (User::const_op_iterator OI = I->op_begin(), OE = I->op_end(); - OI != OE; ++OI) - if (IsPotentialRetainableObjPtr(*OI)) + for (const Use &U : I->operands()) + if (IsPotentialRetainableObjPtr(U)) return ARCInstKind::User; } } diff --git a/llvm/lib/Analysis/OverflowInstAnalysis.cpp b/llvm/lib/Analysis/OverflowInstAnalysis.cpp index 9f17d5b2064d..87a85e6a7364 100644 --- a/llvm/lib/Analysis/OverflowInstAnalysis.cpp +++ b/llvm/lib/Analysis/OverflowInstAnalysis.cpp @@ -69,4 +69,4 @@ bool llvm::isCheckForZeroAndMulWithOverflow(Value *Op0, Value *Op1, bool IsAnd) { Use *Y; return isCheckForZeroAndMulWithOverflow(Op0, Op1, IsAnd, Y); -}
\ No newline at end of file +} diff --git a/llvm/lib/Analysis/PHITransAddr.cpp b/llvm/lib/Analysis/PHITransAddr.cpp index 7f77ab146c4c..c73e1fd82915 100644 --- a/llvm/lib/Analysis/PHITransAddr.cpp +++ b/llvm/lib/Analysis/PHITransAddr.cpp @@ -226,8 +226,8 @@ Value *PHITransAddr::PHITranslateSubExpr(Value *V, BasicBlock *CurBB, return GEP; // Simplify the GEP to handle 'gep x, 0' -> x etc. - if (Value *V = SimplifyGEPInst(GEP->getSourceElementType(), - GEPOps, {DL, TLI, DT, AC})) { + if (Value *V = SimplifyGEPInst(GEP->getSourceElementType(), GEPOps, + GEP->isInBounds(), {DL, TLI, DT, AC})) { for (unsigned i = 0, e = GEPOps.size(); i != e; ++i) RemoveInstInputs(GEPOps[i], InstInputs); diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp index 6dda0bf0a1b4..268ed9d04741 100644 --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -103,7 +103,7 @@ bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) const { // FIXME: The heuristic used below for determining hotness is based on // preliminary SPEC tuning for inliner. This will eventually be a // convenience method that calls isHotCount. - return FunctionCount && isHotCount(FunctionCount.getCount()); + return FunctionCount && isHotCount(FunctionCount->getCount()); } /// Returns true if the function contains hot code. This can include a hot @@ -116,7 +116,7 @@ bool ProfileSummaryInfo::isFunctionHotInCallGraph( if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) - if (isHotCount(FunctionCount.getCount())) + if (isHotCount(FunctionCount->getCount())) return true; if (hasSampleProfile()) { @@ -145,7 +145,7 @@ bool ProfileSummaryInfo::isFunctionColdInCallGraph( if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) - if (!isColdCount(FunctionCount.getCount())) + if (!isColdCount(FunctionCount->getCount())) return false; if (hasSampleProfile()) { @@ -176,10 +176,10 @@ bool ProfileSummaryInfo::isFunctionHotOrColdInCallGraphNthPercentile( return false; if (auto FunctionCount = F->getEntryCount()) { if (isHot && - isHotCountNthPercentile(PercentileCutoff, FunctionCount.getCount())) + isHotCountNthPercentile(PercentileCutoff, FunctionCount->getCount())) return true; if (!isHot && - !isColdCountNthPercentile(PercentileCutoff, FunctionCount.getCount())) + !isColdCountNthPercentile(PercentileCutoff, FunctionCount->getCount())) return false; } if (hasSampleProfile()) { @@ -230,7 +230,7 @@ bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const { // FIXME: The heuristic used below for determining coldness is based on // preliminary SPEC tuning for inliner. This will eventually be a // convenience method that calls isHotCount. - return FunctionCount && isColdCount(FunctionCount.getCount()); + return FunctionCount && isColdCount(FunctionCount->getCount()); } /// Compute the hot and cold thresholds. @@ -316,11 +316,11 @@ bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, } uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const { - return HotCountThreshold ? HotCountThreshold.getValue() : UINT64_MAX; + return HotCountThreshold.getValueOr(UINT64_MAX); } uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const { - return ColdCountThreshold ? ColdCountThreshold.getValue() : 0; + return ColdCountThreshold.getValueOr(0); } bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, diff --git a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp index b9dac2f3ff11..f83d8b0fd230 100644 --- a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp @@ -17,18 +17,21 @@ #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/LineIterator.h" +#include <memory> using namespace llvm; -#define DEBUG_TYPE "inline-replay" +#define DEBUG_TYPE "replay-inline" ReplayInlineAdvisor::ReplayInlineAdvisor( Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, - std::unique_ptr<InlineAdvisor> OriginalAdvisor, StringRef RemarksFile, - bool EmitRemarks) + std::unique_ptr<InlineAdvisor> OriginalAdvisor, + const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks) : InlineAdvisor(M, FAM), OriginalAdvisor(std::move(OriginalAdvisor)), - HasReplayRemarks(false), EmitRemarks(EmitRemarks) { - auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(RemarksFile); + HasReplayRemarks(false), ReplaySettings(ReplaySettings), + EmitRemarks(EmitRemarks) { + + auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(ReplaySettings.ReplayFile); std::error_code EC = BufferOrErr.getError(); if (EC) { Context.emitError("Could not open remarks file: " + EC.message()); @@ -36,47 +39,112 @@ ReplayInlineAdvisor::ReplayInlineAdvisor( } // Example for inline remarks to parse: - // main:3:1.1: _Z3subii inlined into main at callsite sum:1 @ main:3:1.1 + // main:3:1.1: '_Z3subii' inlined into 'main' at callsite sum:1 @ + // main:3:1.1; // We use the callsite string after `at callsite` to replay inlining. line_iterator LineIt(*BufferOrErr.get(), /*SkipBlanks=*/true); + const std::string PositiveRemark = "' inlined into '"; + const std::string NegativeRemark = "' will not be inlined into '"; + for (; !LineIt.is_at_eof(); ++LineIt) { StringRef Line = *LineIt; auto Pair = Line.split(" at callsite "); - auto Callee = Pair.first.split(" inlined into").first.rsplit(": ").second; + bool IsPositiveRemark = true; + if (Pair.first.contains(NegativeRemark)) + IsPositiveRemark = false; + + auto CalleeCaller = + Pair.first.split(IsPositiveRemark ? PositiveRemark : NegativeRemark); + + StringRef Callee = CalleeCaller.first.rsplit(": '").second; + StringRef Caller = CalleeCaller.second.rsplit("'").first; auto CallSite = Pair.second.split(";").first; - if (Callee.empty() || CallSite.empty()) - continue; + if (Callee.empty() || Caller.empty() || CallSite.empty()) { + Context.emitError("Invalid remark format: " + Line); + return; + } std::string Combined = (Callee + CallSite).str(); - InlineSitesFromRemarks.insert(Combined); + InlineSitesFromRemarks[Combined] = IsPositiveRemark; + if (ReplaySettings.ReplayScope == ReplayInlinerSettings::Scope::Function) + CallersToReplay.insert(Caller); } HasReplayRemarks = true; } +std::unique_ptr<InlineAdvisor> llvm::getReplayInlineAdvisor( + Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, + std::unique_ptr<InlineAdvisor> OriginalAdvisor, + const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks) { + auto Advisor = std::make_unique<ReplayInlineAdvisor>( + M, FAM, Context, std::move(OriginalAdvisor), ReplaySettings, EmitRemarks); + if (!Advisor->areReplayRemarksLoaded()) + Advisor.reset(); + return Advisor; +} + std::unique_ptr<InlineAdvice> ReplayInlineAdvisor::getAdviceImpl(CallBase &CB) { assert(HasReplayRemarks); Function &Caller = *CB.getCaller(); auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller); - if (InlineSitesFromRemarks.empty()) - return std::make_unique<DefaultInlineAdvice>(this, CB, None, ORE, - EmitRemarks); + // Decision not made by replay system + if (!hasInlineAdvice(*CB.getFunction())) { + // If there's a registered original advisor, return its decision + if (OriginalAdvisor) + return OriginalAdvisor->getAdvice(CB); - std::string CallSiteLoc = getCallSiteLocation(CB.getDebugLoc()); + // If no decision is made above, return non-decision + return {}; + } + + std::string CallSiteLoc = + formatCallSiteLocation(CB.getDebugLoc(), ReplaySettings.ReplayFormat); StringRef Callee = CB.getCalledFunction()->getName(); std::string Combined = (Callee + CallSiteLoc).str(); - auto Iter = InlineSitesFromRemarks.find(Combined); - Optional<InlineCost> InlineRecommended = None; + // Replay decision, if it has one + auto Iter = InlineSitesFromRemarks.find(Combined); if (Iter != InlineSitesFromRemarks.end()) { - InlineRecommended = llvm::InlineCost::getAlways("found in replay"); + if (InlineSitesFromRemarks[Combined]) { + LLVM_DEBUG(dbgs() << "Replay Inliner: Inlined " << Callee << " @ " + << CallSiteLoc << "\n"); + return std::make_unique<DefaultInlineAdvice>( + this, CB, llvm::InlineCost::getAlways("previously inlined"), ORE, + EmitRemarks); + } else { + LLVM_DEBUG(dbgs() << "Replay Inliner: Not Inlined " << Callee << " @ " + << CallSiteLoc << "\n"); + // A negative inline is conveyed by "None" Optional<InlineCost> + return std::make_unique<DefaultInlineAdvice>(this, CB, None, ORE, + EmitRemarks); + } + } + + // Fallback decisions + if (ReplaySettings.ReplayFallback == + ReplayInlinerSettings::Fallback::AlwaysInline) + return std::make_unique<DefaultInlineAdvice>( + this, CB, llvm::InlineCost::getAlways("AlwaysInline Fallback"), ORE, + EmitRemarks); + else if (ReplaySettings.ReplayFallback == + ReplayInlinerSettings::Fallback::NeverInline) + // A negative inline is conveyed by "None" Optional<InlineCost> + return std::make_unique<DefaultInlineAdvice>(this, CB, None, ORE, + EmitRemarks); + else { + assert(ReplaySettings.ReplayFallback == + ReplayInlinerSettings::Fallback::Original); + // If there's a registered original advisor, return its decision + if (OriginalAdvisor) + return OriginalAdvisor->getAdvice(CB); } - return std::make_unique<DefaultInlineAdvice>(this, CB, InlineRecommended, ORE, - EmitRemarks); + // If no decision is made above, return non-decision + return {}; } diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index f22d834b5e57..f7c22cfb0310 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -139,8 +139,6 @@ using namespace PatternMatch; #define DEBUG_TYPE "scalar-evolution" -STATISTIC(NumArrayLenItCounts, - "Number of trip counts computed with array length"); STATISTIC(NumTripCountsComputed, "Number of loops with predictable loop counts"); STATISTIC(NumTripCountsNotComputed, @@ -1100,7 +1098,7 @@ const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op, SCEV *S = new (SCEVAllocator) SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), Op, IntPtrTy); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Op); return S; } @@ -1220,7 +1218,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty, SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Op); return S; } @@ -1274,7 +1272,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty, SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Op); return S; } @@ -1603,7 +1601,7 @@ ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Op); return S; } @@ -1872,7 +1870,7 @@ ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Op); return S; } @@ -1911,7 +1909,7 @@ ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Op); return S; } @@ -2108,7 +2106,7 @@ ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, { Op }); return S; } @@ -2390,6 +2388,24 @@ StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, } } + // <0,+,nonnegative><nw> is also nuw + // TODO: Add corresponding nsw case + if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, SCEV::FlagNW) && + !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) && Ops.size() == 2 && + Ops[0]->isZero() && IsKnownNonNegative(Ops[1])) + Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); + + // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW + if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) && + Ops.size() == 2) { + if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[0])) + if (UDiv->getOperand(1) == Ops[1]) + Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); + if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[1])) + if (UDiv->getOperand(1) == Ops[0]) + Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); + } + return Flags; } @@ -2449,7 +2465,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, if (Depth > MaxArithDepth || hasHugeExpression(Ops)) return getOrCreateAddExpr(Ops, ComputeFlags(Ops)); - if (SCEV *S = std::get<0>(findExistingSCEVInCache(scAddExpr, Ops))) { + if (SCEV *S = findExistingSCEVInCache(scAddExpr, Ops)) { // Don't strengthen flags if we have no new information. SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S); if (Add->getNoWrapFlags(OrigFlags) != OrigFlags) @@ -2562,8 +2578,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, APInt ConstAdd = C1 + C2; auto AddFlags = AddExpr->getNoWrapFlags(); // Adding a smaller constant is NUW if the original AddExpr was NUW. - if (ScalarEvolution::maskFlags(AddFlags, SCEV::FlagNUW) == - SCEV::FlagNUW && + if (ScalarEvolution::hasFlags(AddFlags, SCEV::FlagNUW) && ConstAdd.ule(C1)) { PreservedFlags = ScalarEvolution::setFlags(PreservedFlags, SCEV::FlagNUW); @@ -2571,8 +2586,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, // Adding a constant with the same sign and small magnitude is NSW, if the // original AddExpr was NSW. - if (ScalarEvolution::maskFlags(AddFlags, SCEV::FlagNSW) == - SCEV::FlagNSW && + if (ScalarEvolution::hasFlags(AddFlags, SCEV::FlagNSW) && C1.isSignBitSet() == ConstAdd.isSignBitSet() && ConstAdd.abs().ule(C1.abs())) { PreservedFlags = @@ -2580,14 +2594,26 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, } if (PreservedFlags != SCEV::FlagAnyWrap) { - SmallVector<const SCEV *, 4> NewOps(AddExpr->op_begin(), - AddExpr->op_end()); + SmallVector<const SCEV *, 4> NewOps(AddExpr->operands()); NewOps[0] = getConstant(ConstAdd); return getAddExpr(NewOps, PreservedFlags); } } } + // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y) + if (Ops.size() == 2) { + const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[0]); + if (Mul && Mul->getNumOperands() == 2 && + Mul->getOperand(0)->isAllOnesValue()) { + const SCEV *X; + const SCEV *Y; + if (matchURem(Mul->getOperand(1), X, Y) && X == Ops[1]) { + return getMulExpr(Y, getUDivExpr(X, Y)); + } + } + } + // Skip past any other cast SCEVs. while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) ++Idx; @@ -2766,7 +2792,8 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, // If we found some loop invariants, fold them into the recurrence. if (!LIOps.empty()) { // Compute nowrap flags for the addition of the loop-invariant ops and - // the addrec. Temporarily push it as an operand for that purpose. + // the addrec. Temporarily push it as an operand for that purpose. These + // flags are valid in the scope of the addrec only. LIOps.push_back(AddRec); SCEV::NoWrapFlags Flags = ComputeFlags(LIOps); LIOps.pop_back(); @@ -2775,10 +2802,25 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, LIOps.push_back(AddRec->getStart()); SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands()); - // This follows from the fact that the no-wrap flags on the outer add - // expression are applicable on the 0th iteration, when the add recurrence - // will be equal to its start value. - AddRecOps[0] = getAddExpr(LIOps, Flags, Depth + 1); + + // It is not in general safe to propagate flags valid on an add within + // the addrec scope to one outside it. We must prove that the inner + // scope is guaranteed to execute if the outer one does to be able to + // safely propagate. We know the program is undefined if poison is + // produced on the inner scoped addrec. We also know that *for this use* + // the outer scoped add can't overflow (because of the flags we just + // computed for the inner scoped add) without the program being undefined. + // Proving that entry to the outer scope neccesitates entry to the inner + // scope, thus proves the program undefined if the flags would be violated + // in the outer scope. + SCEV::NoWrapFlags AddFlags = Flags; + if (AddFlags != SCEV::FlagAnyWrap) { + auto *DefI = getDefiningScopeBound(LIOps); + auto *ReachI = &*AddRecLoop->getHeader()->begin(); + if (!isGuaranteedToTransferExecutionTo(DefI, ReachI)) + AddFlags = SCEV::FlagAnyWrap; + } + AddRecOps[0] = getAddExpr(LIOps, AddFlags, Depth + 1); // Build the new addrec. Propagate the NUW and NSW flags if both the // outer add and the inner addrec are guaranteed to have no overflow. @@ -2862,7 +2904,7 @@ ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops, S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Ops); } S->setNoWrapFlags(Flags); return S; @@ -2885,7 +2927,8 @@ ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops, S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + LoopUsers[L].push_back(S); + registerUser(S, Ops); } setNoWrapFlags(S, Flags); return S; @@ -2907,7 +2950,7 @@ ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops, S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator), O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Ops); } S->setNoWrapFlags(Flags); return S; @@ -3022,7 +3065,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, if (Depth > MaxArithDepth || hasHugeExpression(Ops)) return getOrCreateMulExpr(Ops, ComputeFlags(Ops)); - if (SCEV *S = std::get<0>(findExistingSCEVInCache(scMulExpr, Ops))) { + if (SCEV *S = findExistingSCEVInCache(scMulExpr, Ops)) { // Don't strengthen flags if we have no new information. SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S); if (Mul->getNoWrapFlags(OrigFlags) != OrigFlags) @@ -3416,7 +3459,7 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), LHS, RHS); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, {LHS, RHS}); return S; } @@ -3593,13 +3636,21 @@ ScalarEvolution::getGEPExpr(GEPOperator *GEP, // getSCEV(Base)->getType() has the same address space as Base->getType() // because SCEV::getType() preserves the address space. Type *IntIdxTy = getEffectiveSCEVType(BaseExpr->getType()); - // FIXME(PR23527): Don't blindly transfer the inbounds flag from the GEP - // instruction to its SCEV, because the Instruction may be guarded by control - // flow and the no-overflow bits may not be valid for the expression in any - // context. This can be fixed similarly to how these flags are handled for - // adds. + const bool AssumeInBoundsFlags = [&]() { + if (!GEP->isInBounds()) + return false; + + // We'd like to propagate flags from the IR to the corresponding SCEV nodes, + // but to do that, we have to ensure that said flag is valid in the entire + // defined scope of the SCEV. + auto *GEPI = dyn_cast<Instruction>(GEP); + // TODO: non-instructions have global scope. We might be able to prove + // some global scope cases + return GEPI && isSCEVExprNeverPoison(GEPI); + }(); + SCEV::NoWrapFlags OffsetWrap = - GEP->isInBounds() ? SCEV::FlagNSW : SCEV::FlagAnyWrap; + AssumeInBoundsFlags ? SCEV::FlagNSW : SCEV::FlagAnyWrap; Type *CurTy = GEP->getType(); bool FirstIter = true; @@ -3645,21 +3696,22 @@ ScalarEvolution::getGEPExpr(GEPOperator *GEP, // Add the base address and the offset. We cannot use the nsw flag, as the // base address is unsigned. However, if we know that the offset is // non-negative, we can use nuw. - SCEV::NoWrapFlags BaseWrap = GEP->isInBounds() && isKnownNonNegative(Offset) + SCEV::NoWrapFlags BaseWrap = AssumeInBoundsFlags && isKnownNonNegative(Offset) ? SCEV::FlagNUW : SCEV::FlagAnyWrap; - return getAddExpr(BaseExpr, Offset, BaseWrap); + auto *GEPExpr = getAddExpr(BaseExpr, Offset, BaseWrap); + assert(BaseExpr->getType() == GEPExpr->getType() && + "GEP should not change type mid-flight."); + return GEPExpr; } -std::tuple<SCEV *, FoldingSetNodeID, void *> -ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType, - ArrayRef<const SCEV *> Ops) { +SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType, + ArrayRef<const SCEV *> Ops) { FoldingSetNodeID ID; - void *IP = nullptr; ID.AddInteger(SCEVType); for (unsigned i = 0, e = Ops.size(); i != e; ++i) ID.AddPointer(Ops[i]); - return std::tuple<SCEV *, FoldingSetNodeID, void *>( - UniqueSCEVs.FindNodeOrInsertPos(ID, IP), std::move(ID), IP); + void *IP = nullptr; + return UniqueSCEVs.FindNodeOrInsertPos(ID, IP); } const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) { @@ -3689,7 +3741,7 @@ const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind, GroupByComplexity(Ops, &LI, DT); // Check if we have created the same expression before. - if (const SCEV *S = std::get<0>(findExistingSCEVInCache(Kind, Ops))) { + if (const SCEV *S = findExistingSCEVInCache(Kind, Ops)) { return S; } @@ -3787,10 +3839,12 @@ const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind, // Okay, it looks like we really DO need an expr. Check to see if we // already have one, otherwise create a new one. - const SCEV *ExistingSCEV; FoldingSetNodeID ID; - void *IP; - std::tie(ExistingSCEV, ID, IP) = findExistingSCEVInCache(Kind, Ops); + ID.AddInteger(Kind); + for (unsigned i = 0, e = Ops.size(); i != e; ++i) + ID.AddPointer(Ops[i]); + void *IP = nullptr; + const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP); if (ExistingSCEV) return ExistingSCEV; const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); @@ -3799,7 +3853,7 @@ const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind, SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); + registerUser(S, Ops); return S; } @@ -3943,6 +3997,21 @@ Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const { return getTypeSizeInBits(T1) >= getTypeSizeInBits(T2) ? T1 : T2; } +bool ScalarEvolution::instructionCouldExistWitthOperands(const SCEV *A, + const SCEV *B) { + /// For a valid use point to exist, the defining scope of one operand + /// must dominate the other. + bool PreciseA, PreciseB; + auto *ScopeA = getDefiningScopeBound({A}, PreciseA); + auto *ScopeB = getDefiningScopeBound({B}, PreciseB); + if (!PreciseA || !PreciseB) + // Can't tell. + return false; + return (ScopeA == ScopeB) || DT.dominates(ScopeA, ScopeB) || + DT.dominates(ScopeB, ScopeA); +} + + const SCEV *ScalarEvolution::getCouldNotCompute() { return CouldNotCompute.get(); } @@ -4025,24 +4094,6 @@ void ScalarEvolution::eraseValueFromMap(Value *V) { } } -/// Check whether value has nuw/nsw/exact set but SCEV does not. -/// TODO: In reality it is better to check the poison recursively -/// but this is better than nothing. -static bool SCEVLostPoisonFlags(const SCEV *S, const Value *V) { - if (auto *I = dyn_cast<Instruction>(V)) { - if (isa<OverflowingBinaryOperator>(I)) { - if (auto *NS = dyn_cast<SCEVNAryExpr>(S)) { - if (I->hasNoSignedWrap() && !NS->hasNoSignedWrap()) - return true; - if (I->hasNoUnsignedWrap() && !NS->hasNoUnsignedWrap()) - return true; - } - } else if (isa<PossiblyExactOperator>(I) && I->isExact()) - return true; - } - return false; -} - /// Return an existing SCEV if it exists, otherwise analyze the expression and /// create a new one. const SCEV *ScalarEvolution::getSCEV(Value *V) { @@ -4056,7 +4107,7 @@ const SCEV *ScalarEvolution::getSCEV(Value *V) { // ValueExprMap before insert S->{V, 0} into ExprValueMap. std::pair<ValueExprMapType::iterator, bool> Pair = ValueExprMap.insert({SCEVCallbackVH(V, this), S}); - if (Pair.second && !SCEVLostPoisonFlags(S, V)) { + if (Pair.second) { ExprValueMap[S].insert({V, nullptr}); // If S == Stripped + Offset, add Stripped -> {V, Offset} into @@ -4120,6 +4171,8 @@ static const SCEV *MatchNotExpr(const SCEV *Expr) { /// Return a SCEV corresponding to ~V = -1-V const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { + assert(!V->getType()->isPointerTy() && "Can't negate pointer"); + if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) return getConstant( cast<ConstantInt>(ConstantExpr::getNot(VC->getValue()))); @@ -4146,17 +4199,16 @@ const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { return getMinusSCEV(getMinusOne(Ty), V); } -/// Compute an expression equivalent to S - getPointerBase(S). -static const SCEV *removePointerBase(ScalarEvolution *SE, const SCEV *P) { +const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) { assert(P->getType()->isPointerTy()); if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(P)) { // The base of an AddRec is the first operand. SmallVector<const SCEV *> Ops{AddRec->operands()}; - Ops[0] = removePointerBase(SE, Ops[0]); + Ops[0] = removePointerBase(Ops[0]); // Don't try to transfer nowrap flags for now. We could in some cases // (for example, if pointer operand of the AddRec is a SCEVUnknown). - return SE->getAddRecExpr(Ops, AddRec->getLoop(), SCEV::FlagAnyWrap); + return getAddRecExpr(Ops, AddRec->getLoop(), SCEV::FlagAnyWrap); } if (auto *Add = dyn_cast<SCEVAddExpr>(P)) { // The base of an Add is the pointer operand. @@ -4164,21 +4216,17 @@ static const SCEV *removePointerBase(ScalarEvolution *SE, const SCEV *P) { const SCEV **PtrOp = nullptr; for (const SCEV *&AddOp : Ops) { if (AddOp->getType()->isPointerTy()) { - // If we find an Add with multiple pointer operands, treat it as a - // pointer base to be consistent with getPointerBase. Eventually - // we should be able to assert this is impossible. - if (PtrOp) - return SE->getZero(P->getType()); + assert(!PtrOp && "Cannot have multiple pointer ops"); PtrOp = &AddOp; } } - *PtrOp = removePointerBase(SE, *PtrOp); + *PtrOp = removePointerBase(*PtrOp); // Don't try to transfer nowrap flags for now. We could in some cases // (for example, if the pointer operand of the Add is a SCEVUnknown). - return SE->getAddExpr(Ops); + return getAddExpr(Ops); } // Any other expression must be a pointer base. - return SE->getZero(P->getType()); + return getZero(P->getType()); } const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, @@ -4195,8 +4243,8 @@ const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, if (!LHS->getType()->isPointerTy() || getPointerBase(LHS) != getPointerBase(RHS)) return getCouldNotCompute(); - LHS = removePointerBase(this, LHS); - RHS = removePointerBase(this, RHS); + LHS = removePointerBase(LHS); + RHS = removePointerBase(RHS); } // We represent LHS - RHS as LHS + (-1)*RHS. This transformation @@ -4204,7 +4252,7 @@ const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, auto AddFlags = SCEV::FlagAnyWrap; const bool RHSIsNotMinSigned = !getSignedRangeMin(RHS).isMinSignedValue(); - if (maskFlags(Flags, SCEV::FlagNSW) == SCEV::FlagNSW) { + if (hasFlags(Flags, SCEV::FlagNSW)) { // Let M be the minimum representable signed value. Then (-1)*RHS // signed-wraps if and only if RHS is M. That can happen even for // a NSW subtraction because e.g. (-1)*M signed-wraps even though @@ -4359,14 +4407,11 @@ const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { const SCEV *PtrOp = nullptr; for (const SCEV *AddOp : Add->operands()) { if (AddOp->getType()->isPointerTy()) { - // Cannot find the base of an expression with multiple pointer ops. - if (PtrOp) - return V; + assert(!PtrOp && "Cannot have multiple pointer ops"); PtrOp = AddOp; } } - if (!PtrOp) // All operands were non-pointer. - return V; + assert(PtrOp && "Must have pointer op"); V = PtrOp; } else // Not something we can look further into. return V; @@ -4374,24 +4419,25 @@ const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { } /// Push users of the given Instruction onto the given Worklist. -static void -PushDefUseChildren(Instruction *I, - SmallVectorImpl<Instruction *> &Worklist) { +static void PushDefUseChildren(Instruction *I, + SmallVectorImpl<Instruction *> &Worklist, + SmallPtrSetImpl<Instruction *> &Visited) { // Push the def-use children onto the Worklist stack. - for (User *U : I->users()) - Worklist.push_back(cast<Instruction>(U)); + for (User *U : I->users()) { + auto *UserInsn = cast<Instruction>(U); + if (Visited.insert(UserInsn).second) + Worklist.push_back(UserInsn); + } } void ScalarEvolution::forgetSymbolicName(Instruction *PN, const SCEV *SymName) { SmallVector<Instruction *, 16> Worklist; - PushDefUseChildren(PN, Worklist); - SmallPtrSet<Instruction *, 8> Visited; + SmallVector<const SCEV *, 8> ToForget; Visited.insert(PN); + Worklist.push_back(PN); while (!Worklist.empty()) { Instruction *I = Worklist.pop_back_val(); - if (!Visited.insert(I).second) - continue; auto It = ValueExprMap.find_as(static_cast<Value *>(I)); if (It != ValueExprMap.end()) { @@ -4413,12 +4459,13 @@ void ScalarEvolution::forgetSymbolicName(Instruction *PN, const SCEV *SymName) { !isa<SCEVUnknown>(Old) || (I != PN && Old == SymName)) { eraseValueFromMap(It->first); - forgetMemoizedResults(Old); + ToForget.push_back(Old); } } - PushDefUseChildren(I, Worklist); + PushDefUseChildren(I, Worklist, Visited); } + forgetMemoizedResults(ToForget); } namespace { @@ -6109,7 +6156,7 @@ ScalarEvolution::getRangeRef(const SCEV *S, // initial value. if (AddRec->hasNoUnsignedWrap()) { APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart()); - if (!UnsignedMinValue.isNullValue()) + if (!UnsignedMinValue.isZero()) ConservativeResult = ConservativeResult.intersectWith( ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType); } @@ -6211,9 +6258,9 @@ ScalarEvolution::getRangeRef(const SCEV *S, if (NS > 1) { // If we know any of the sign bits, we know all of the sign bits. - if (!Known.Zero.getHiBits(NS).isNullValue()) + if (!Known.Zero.getHiBits(NS).isZero()) Known.Zero.setHighBits(NS); - if (!Known.One.getHiBits(NS).isNullValue()) + if (!Known.One.getHiBits(NS).isZero()) Known.One.setHighBits(NS); } @@ -6549,17 +6596,99 @@ SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) { return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagAnyWrap; } -bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) { - // Here we check that I is in the header of the innermost loop containing I, - // since we only deal with instructions in the loop header. The actual loop we - // need to check later will come from an add recurrence, but getting that - // requires computing the SCEV of the operands, which can be expensive. This - // check we can do cheaply to rule out some cases early. - Loop *InnermostContainingLoop = LI.getLoopFor(I->getParent()); - if (InnermostContainingLoop == nullptr || - InnermostContainingLoop->getHeader() != I->getParent()) - return false; +const Instruction * +ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) { + if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S)) + return &*AddRec->getLoop()->getHeader()->begin(); + if (auto *U = dyn_cast<SCEVUnknown>(S)) + if (auto *I = dyn_cast<Instruction>(U->getValue())) + return I; + return nullptr; +} +/// Fills \p Ops with unique operands of \p S, if it has operands. If not, +/// \p Ops remains unmodified. +static void collectUniqueOps(const SCEV *S, + SmallVectorImpl<const SCEV *> &Ops) { + SmallPtrSet<const SCEV *, 4> Unique; + auto InsertUnique = [&](const SCEV *S) { + if (Unique.insert(S).second) + Ops.push_back(S); + }; + if (auto *S2 = dyn_cast<SCEVCastExpr>(S)) + for (auto *Op : S2->operands()) + InsertUnique(Op); + else if (auto *S2 = dyn_cast<SCEVNAryExpr>(S)) + for (auto *Op : S2->operands()) + InsertUnique(Op); + else if (auto *S2 = dyn_cast<SCEVUDivExpr>(S)) + for (auto *Op : S2->operands()) + InsertUnique(Op); +} + +const Instruction * +ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops, + bool &Precise) { + Precise = true; + // Do a bounded search of the def relation of the requested SCEVs. + SmallSet<const SCEV *, 16> Visited; + SmallVector<const SCEV *> Worklist; + auto pushOp = [&](const SCEV *S) { + if (!Visited.insert(S).second) + return; + // Threshold of 30 here is arbitrary. + if (Visited.size() > 30) { + Precise = false; + return; + } + Worklist.push_back(S); + }; + + for (auto *S : Ops) + pushOp(S); + + const Instruction *Bound = nullptr; + while (!Worklist.empty()) { + auto *S = Worklist.pop_back_val(); + if (auto *DefI = getNonTrivialDefiningScopeBound(S)) { + if (!Bound || DT.dominates(Bound, DefI)) + Bound = DefI; + } else { + SmallVector<const SCEV *, 4> Ops; + collectUniqueOps(S, Ops); + for (auto *Op : Ops) + pushOp(Op); + } + } + return Bound ? Bound : &*F.getEntryBlock().begin(); +} + +const Instruction * +ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops) { + bool Discard; + return getDefiningScopeBound(Ops, Discard); +} + +bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A, + const Instruction *B) { + if (A->getParent() == B->getParent() && + isGuaranteedToTransferExecutionToSuccessor(A->getIterator(), + B->getIterator())) + return true; + + auto *BLoop = LI.getLoopFor(B->getParent()); + if (BLoop && BLoop->getHeader() == B->getParent() && + BLoop->getLoopPreheader() == A->getParent() && + isGuaranteedToTransferExecutionToSuccessor(A->getIterator(), + A->getParent()->end()) && + isGuaranteedToTransferExecutionToSuccessor(B->getParent()->begin(), + B->getIterator())) + return true; + return false; +} + + +bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) { // Only proceed if we can prove that I does not yield poison. if (!programUndefinedIfPoison(I)) return false; @@ -6570,39 +6699,20 @@ bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) { // instructions can map to the same SCEV. If we apply NSW or NUW from I to // the SCEV, we must guarantee no wrapping for that SCEV also when it is // derived from other instructions that map to the same SCEV. We cannot make - // that guarantee for cases where I is not executed. So we need to find the - // loop that I is considered in relation to and prove that I is executed for - // every iteration of that loop. That implies that the value that I - // calculates does not wrap anywhere in the loop, so then we can apply the - // flags to the SCEV. - // - // We check isLoopInvariant to disambiguate in case we are adding recurrences - // from different loops, so that we know which loop to prove that I is - // executed in. - for (unsigned OpIndex = 0; OpIndex < I->getNumOperands(); ++OpIndex) { + // that guarantee for cases where I is not executed. So we need to find a + // upper bound on the defining scope for the SCEV, and prove that I is + // executed every time we enter that scope. When the bounding scope is a + // loop (the common case), this is equivalent to proving I executes on every + // iteration of that loop. + SmallVector<const SCEV *> SCEVOps; + for (const Use &Op : I->operands()) { // I could be an extractvalue from a call to an overflow intrinsic. // TODO: We can do better here in some cases. - if (!isSCEVable(I->getOperand(OpIndex)->getType())) - return false; - const SCEV *Op = getSCEV(I->getOperand(OpIndex)); - if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { - bool AllOtherOpsLoopInvariant = true; - for (unsigned OtherOpIndex = 0; OtherOpIndex < I->getNumOperands(); - ++OtherOpIndex) { - if (OtherOpIndex != OpIndex) { - const SCEV *OtherOp = getSCEV(I->getOperand(OtherOpIndex)); - if (!isLoopInvariant(OtherOp, AddRec->getLoop())) { - AllOtherOpsLoopInvariant = false; - break; - } - } - } - if (AllOtherOpsLoopInvariant && - isGuaranteedToExecuteForEveryIteration(I, AddRec->getLoop())) - return true; - } + if (isSCEVable(Op->getType())) + SCEVOps.push_back(getSCEV(Op)); } - return false; + auto *DefI = getDefiningScopeBound(SCEVOps); + return isGuaranteedToTransferExecutionTo(DefI, I); } bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) { @@ -7144,10 +7254,21 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { // Iteration Count Computation Code // -const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) { - // Get the trip count from the BE count by adding 1. Overflow, results - // in zero which means "unknown". - return getAddExpr(ExitCount, getOne(ExitCount->getType())); +const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount, + bool Extend) { + if (isa<SCEVCouldNotCompute>(ExitCount)) + return getCouldNotCompute(); + + auto *ExitCountType = ExitCount->getType(); + assert(ExitCountType->isIntegerTy()); + + if (!Extend) + return getAddExpr(ExitCount, getOne(ExitCountType)); + + auto *WiderType = Type::getIntNTy(ExitCountType->getContext(), + 1 + ExitCountType->getScalarSizeInBits()); + return getAddExpr(getNoopOrZeroExtend(ExitCount, WiderType), + getOne(WiderType)); } static unsigned getConstantTripCount(const SCEVConstant *ExitCount) { @@ -7186,6 +7307,131 @@ unsigned ScalarEvolution::getSmallConstantMaxTripCount(const Loop *L) { return getConstantTripCount(MaxExitCount); } +const SCEV *ScalarEvolution::getConstantMaxTripCountFromArray(const Loop *L) { + // We can't infer from Array in Irregular Loop. + // FIXME: It's hard to infer loop bound from array operated in Nested Loop. + if (!L->isLoopSimplifyForm() || !L->isInnermost()) + return getCouldNotCompute(); + + // FIXME: To make the scene more typical, we only analysis loops that have + // one exiting block and that block must be the latch. To make it easier to + // capture loops that have memory access and memory access will be executed + // in each iteration. + const BasicBlock *LoopLatch = L->getLoopLatch(); + assert(LoopLatch && "See defination of simplify form loop."); + if (L->getExitingBlock() != LoopLatch) + return getCouldNotCompute(); + + const DataLayout &DL = getDataLayout(); + SmallVector<const SCEV *> InferCountColl; + for (auto *BB : L->getBlocks()) { + // Go here, we can know that Loop is a single exiting and simplified form + // loop. Make sure that infer from Memory Operation in those BBs must be + // executed in loop. First step, we can make sure that max execution time + // of MemAccessBB in loop represents latch max excution time. + // If MemAccessBB does not dom Latch, skip. + // Entry + // │ + // ┌─────▼─────┐ + // │Loop Header◄─────┐ + // └──┬──────┬─┘ │ + // │ │ │ + // ┌────────▼──┐ ┌─▼─────┐ │ + // │MemAccessBB│ │OtherBB│ │ + // └────────┬──┘ └─┬─────┘ │ + // │ │ │ + // ┌─▼──────▼─┐ │ + // │Loop Latch├─────┘ + // └────┬─────┘ + // ▼ + // Exit + if (!DT.dominates(BB, LoopLatch)) + continue; + + for (Instruction &Inst : *BB) { + // Find Memory Operation Instruction. + auto *GEP = getLoadStorePointerOperand(&Inst); + if (!GEP) + continue; + + auto *ElemSize = dyn_cast<SCEVConstant>(getElementSize(&Inst)); + // Do not infer from scalar type, eg."ElemSize = sizeof()". + if (!ElemSize) + continue; + + // Use a existing polynomial recurrence on the trip count. + auto *AddRec = dyn_cast<SCEVAddRecExpr>(getSCEV(GEP)); + if (!AddRec) + continue; + auto *ArrBase = dyn_cast<SCEVUnknown>(getPointerBase(AddRec)); + auto *Step = dyn_cast<SCEVConstant>(AddRec->getStepRecurrence(*this)); + if (!ArrBase || !Step) + continue; + assert(isLoopInvariant(ArrBase, L) && "See addrec definition"); + + // Only handle { %array + step }, + // FIXME: {(SCEVAddRecExpr) + step } could not be analysed here. + if (AddRec->getStart() != ArrBase) + continue; + + // Memory operation pattern which have gaps. + // Or repeat memory opreation. + // And index of GEP wraps arround. + if (Step->getAPInt().getActiveBits() > 32 || + Step->getAPInt().getZExtValue() != + ElemSize->getAPInt().getZExtValue() || + Step->isZero() || Step->getAPInt().isNegative()) + continue; + + // Only infer from stack array which has certain size. + // Make sure alloca instruction is not excuted in loop. + AllocaInst *AllocateInst = dyn_cast<AllocaInst>(ArrBase->getValue()); + if (!AllocateInst || L->contains(AllocateInst->getParent())) + continue; + + // Make sure only handle normal array. + auto *Ty = dyn_cast<ArrayType>(AllocateInst->getAllocatedType()); + auto *ArrSize = dyn_cast<ConstantInt>(AllocateInst->getArraySize()); + if (!Ty || !ArrSize || !ArrSize->isOne()) + continue; + // Also make sure step was increased the same with sizeof allocated + // element type. + const PointerType *GEPT = dyn_cast<PointerType>(GEP->getType()); + if (Ty->getElementType() != GEPT->getElementType()) + continue; + + // FIXME: Since gep indices are silently zext to the indexing type, + // we will have a narrow gep index which wraps around rather than + // increasing strictly, we shoule ensure that step is increasing + // strictly by the loop iteration. + // Now we can infer a max execution time by MemLength/StepLength. + const SCEV *MemSize = + getConstant(Step->getType(), DL.getTypeAllocSize(Ty)); + auto *MaxExeCount = + dyn_cast<SCEVConstant>(getUDivCeilSCEV(MemSize, Step)); + if (!MaxExeCount || MaxExeCount->getAPInt().getActiveBits() > 32) + continue; + + // If the loop reaches the maximum number of executions, we can not + // access bytes starting outside the statically allocated size without + // being immediate UB. But it is allowed to enter loop header one more + // time. + auto *InferCount = dyn_cast<SCEVConstant>( + getAddExpr(MaxExeCount, getOne(MaxExeCount->getType()))); + // Discard the maximum number of execution times under 32bits. + if (!InferCount || InferCount->getAPInt().getActiveBits() > 32) + continue; + + InferCountColl.push_back(InferCount); + } + } + + if (InferCountColl.size() == 0) + return getCouldNotCompute(); + + return getUMinFromMismatchedTypes(InferCountColl); +} + unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) { SmallVector<BasicBlock *, 8> ExitingBlocks; L->getExitingBlocks(ExitingBlocks); @@ -7287,13 +7533,15 @@ bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) { } /// Push PHI nodes in the header of the given loop onto the given Worklist. -static void -PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) { +static void PushLoopPHIs(const Loop *L, + SmallVectorImpl<Instruction *> &Worklist, + SmallPtrSetImpl<Instruction *> &Visited) { BasicBlock *Header = L->getHeader(); // Push all Loop-header PHIs onto the Worklist stack. for (PHINode &PN : Header->phis()) - Worklist.push_back(&PN); + if (Visited.insert(&PN).second) + Worklist.push_back(&PN); } const ScalarEvolution::BackedgeTakenInfo & @@ -7354,9 +7602,9 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { // it handles SCEVUnknown PHI nodes specially. if (Result.hasAnyInfo()) { SmallVector<Instruction *, 16> Worklist; - PushLoopPHIs(L, Worklist); - SmallPtrSet<Instruction *, 8> Discovered; + SmallVector<const SCEV *, 8> ToForget; + PushLoopPHIs(L, Worklist, Discovered); while (!Worklist.empty()) { Instruction *I = Worklist.pop_back_val(); @@ -7373,7 +7621,7 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { // own when it gets to that point. if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) { eraseValueFromMap(It->first); - forgetMemoizedResults(Old); + ToForget.push_back(Old); } if (PHINode *PN = dyn_cast<PHINode>(I)) ConstantEvolutionLoopExitValue.erase(PN); @@ -7405,6 +7653,7 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { Worklist.push_back(I); } } + forgetMemoizedResults(ToForget); } // Re-lookup the insert position, since the call to @@ -7441,6 +7690,7 @@ void ScalarEvolution::forgetLoop(const Loop *L) { SmallVector<const Loop *, 16> LoopWorklist(1, L); SmallVector<Instruction *, 32> Worklist; SmallPtrSet<Instruction *, 16> Visited; + SmallVector<const SCEV *, 16> ToForget; // Iterate over all the loops and sub-loops to drop SCEV information. while (!LoopWorklist.empty()) { @@ -7462,29 +7712,27 @@ void ScalarEvolution::forgetLoop(const Loop *L) { auto LoopUsersItr = LoopUsers.find(CurrL); if (LoopUsersItr != LoopUsers.end()) { - for (auto *S : LoopUsersItr->second) - forgetMemoizedResults(S); + ToForget.insert(ToForget.end(), LoopUsersItr->second.begin(), + LoopUsersItr->second.end()); LoopUsers.erase(LoopUsersItr); } // Drop information about expressions based on loop-header PHIs. - PushLoopPHIs(CurrL, Worklist); + PushLoopPHIs(CurrL, Worklist, Visited); while (!Worklist.empty()) { Instruction *I = Worklist.pop_back_val(); - if (!Visited.insert(I).second) - continue; ValueExprMapType::iterator It = ValueExprMap.find_as(static_cast<Value *>(I)); if (It != ValueExprMap.end()) { eraseValueFromMap(It->first); - forgetMemoizedResults(It->second); + ToForget.push_back(It->second); if (PHINode *PN = dyn_cast<PHINode>(I)) ConstantEvolutionLoopExitValue.erase(PN); } - PushDefUseChildren(I, Worklist); + PushDefUseChildren(I, Worklist, Visited); } LoopPropertiesCache.erase(CurrL); @@ -7492,6 +7740,7 @@ void ScalarEvolution::forgetLoop(const Loop *L) { // ValuesAtScopes map. LoopWorklist.append(CurrL->begin(), CurrL->end()); } + forgetMemoizedResults(ToForget); } void ScalarEvolution::forgetTopmostLoop(const Loop *L) { @@ -7506,25 +7755,25 @@ void ScalarEvolution::forgetValue(Value *V) { // Drop information about expressions based on loop-header PHIs. SmallVector<Instruction *, 16> Worklist; + SmallPtrSet<Instruction *, 8> Visited; + SmallVector<const SCEV *, 8> ToForget; Worklist.push_back(I); + Visited.insert(I); - SmallPtrSet<Instruction *, 8> Visited; while (!Worklist.empty()) { I = Worklist.pop_back_val(); - if (!Visited.insert(I).second) - continue; - ValueExprMapType::iterator It = ValueExprMap.find_as(static_cast<Value *>(I)); if (It != ValueExprMap.end()) { eraseValueFromMap(It->first); - forgetMemoizedResults(It->second); + ToForget.push_back(It->second); if (PHINode *PN = dyn_cast<PHINode>(I)) ConstantEvolutionLoopExitValue.erase(PN); } - PushDefUseChildren(I, Worklist); + PushDefUseChildren(I, Worklist, Visited); } + forgetMemoizedResults(ToForget); } void ScalarEvolution::forgetLoopDispositions(const Loop *L) { @@ -7598,7 +7847,7 @@ ScalarEvolution::BackedgeTakenInfo::getConstantMax(ScalarEvolution *SE) const { return !ENT.hasAlwaysTruePredicate(); }; - if (any_of(ExitNotTaken, PredicateNotAlwaysTrue) || !getConstantMax()) + if (!getConstantMax() || any_of(ExitNotTaken, PredicateNotAlwaysTrue)) return SE->getCouldNotCompute(); assert((isa<SCEVCouldNotCompute>(getConstantMax()) || @@ -7635,6 +7884,12 @@ ScalarEvolution::ExitLimit::ExitLimit( const SCEV *E, const SCEV *M, bool MaxOrZero, ArrayRef<const SmallPtrSetImpl<const SCEVPredicate *> *> PredSetList) : ExactNotTaken(E), MaxNotTaken(M), MaxOrZero(MaxOrZero) { + // If we prove the max count is zero, so is the symbolic bound. This happens + // in practice due to differences in a) how context sensitive we've chosen + // to be and b) how we reason about bounds impied by UB. + if (MaxNotTaken->isZero()) + ExactNotTaken = MaxNotTaken; + assert((isa<SCEVCouldNotCompute>(ExactNotTaken) || !isa<SCEVCouldNotCompute>(MaxNotTaken)) && "Exact is not allowed to be less precise than Max"); @@ -7740,7 +7995,7 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, if (auto *BI = dyn_cast<BranchInst>(ExitBB->getTerminator())) if (auto *CI = dyn_cast<ConstantInt>(BI->getCondition())) { bool ExitIfTrue = !L->contains(BI->getSuccessor(0)); - if ((ExitIfTrue && CI->isZero()) || (!ExitIfTrue && CI->isOne())) + if (ExitIfTrue == CI->isZero()) continue; } @@ -8030,15 +8285,6 @@ ScalarEvolution::computeExitLimitFromICmp(const Loop *L, Pred = ExitCond->getInversePredicate(); const ICmpInst::Predicate OriginalPred = Pred; - // Handle common loops like: for (X = "string"; *X; ++X) - if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0))) - if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) { - ExitLimit ItCnt = - computeLoadConstantCompareExitLimit(LI, RHS, L, Pred); - if (ItCnt.hasAnyInfo()) - return ItCnt; - } - const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); @@ -8070,6 +8316,32 @@ ScalarEvolution::computeExitLimitFromICmp(const Loop *L, if (!isa<SCEVCouldNotCompute>(Ret)) return Ret; } + // If this loop must exit based on this condition (or execute undefined + // behaviour), and we can prove the test sequence produced must repeat + // the same values on self-wrap of the IV, then we can infer that IV + // doesn't self wrap because if it did, we'd have an infinite (undefined) + // loop. + if (ControlsExit && isLoopInvariant(RHS, L) && loopHasNoAbnormalExits(L) && + loopIsFiniteByAssumption(L)) { + + // TODO: We can peel off any functions which are invertible *in L*. Loop + // invariant terms are effectively constants for our purposes here. + auto *InnerLHS = LHS; + if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS)) + InnerLHS = ZExt->getOperand(); + if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(InnerLHS)) { + auto *StrideC = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this)); + if (!AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() && + StrideC && StrideC->getAPInt().isPowerOf2()) { + auto Flags = AR->getNoWrapFlags(); + Flags = setFlags(Flags, SCEV::FlagNW); + SmallVector<const SCEV*> Operands{AR->operands()}; + Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags); + setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags); + } + } + } + switch (Pred) { case ICmpInst::ICMP_NE: { // while (X != Y) // Convert to: while (X-Y != 0) @@ -8169,85 +8441,6 @@ EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, return cast<SCEVConstant>(Val)->getValue(); } -/// Given an exit condition of 'icmp op load X, cst', try to see if we can -/// compute the backedge execution count. -ScalarEvolution::ExitLimit -ScalarEvolution::computeLoadConstantCompareExitLimit( - LoadInst *LI, - Constant *RHS, - const Loop *L, - ICmpInst::Predicate predicate) { - if (LI->isVolatile()) return getCouldNotCompute(); - - // Check to see if the loaded pointer is a getelementptr of a global. - // TODO: Use SCEV instead of manually grubbing with GEPs. - GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)); - if (!GEP) return getCouldNotCompute(); - - // Make sure that it is really a constant global we are gepping, with an - // initializer, and make sure the first IDX is really 0. - GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)); - if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || - GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) || - !cast<Constant>(GEP->getOperand(1))->isNullValue()) - return getCouldNotCompute(); - - // Okay, we allow one non-constant index into the GEP instruction. - Value *VarIdx = nullptr; - std::vector<Constant*> Indexes; - unsigned VarIdxNum = 0; - for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i) - if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { - Indexes.push_back(CI); - } else if (!isa<ConstantInt>(GEP->getOperand(i))) { - if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's. - VarIdx = GEP->getOperand(i); - VarIdxNum = i-2; - Indexes.push_back(nullptr); - } - - // Loop-invariant loads may be a byproduct of loop optimization. Skip them. - if (!VarIdx) - return getCouldNotCompute(); - - // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant. - // Check to see if X is a loop variant variable value now. - const SCEV *Idx = getSCEV(VarIdx); - Idx = getSCEVAtScope(Idx, L); - - // We can only recognize very limited forms of loop index expressions, in - // particular, only affine AddRec's like {C1,+,C2}<L>. - const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx); - if (!IdxExpr || IdxExpr->getLoop() != L || !IdxExpr->isAffine() || - isLoopInvariant(IdxExpr, L) || - !isa<SCEVConstant>(IdxExpr->getOperand(0)) || - !isa<SCEVConstant>(IdxExpr->getOperand(1))) - return getCouldNotCompute(); - - unsigned MaxSteps = MaxBruteForceIterations; - for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) { - ConstantInt *ItCst = ConstantInt::get( - cast<IntegerType>(IdxExpr->getType()), IterationNum); - ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this); - - // Form the GEP offset. - Indexes[VarIdxNum] = Val; - - Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(), - Indexes); - if (!Result) break; // Cannot compute! - - // Evaluate the condition for this iteration. - Result = ConstantExpr::getICmp(predicate, Result, RHS); - if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure - if (cast<ConstantInt>(Result)->getValue().isMinValue()) { - ++NumArrayLenItCounts; - return getConstant(ItCst); // Found terminating iteration! - } - } - return getCouldNotCompute(); -} - ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit( Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) { ConstantInt *RHS = dyn_cast<ConstantInt>(RHSV); @@ -9160,7 +9353,7 @@ GetQuadraticEquation(const SCEVAddRecExpr *AddRec) { APInt L = LC->getAPInt(); APInt M = MC->getAPInt(); APInt N = NC->getAPInt(); - assert(!N.isNullValue() && "This is not a quadratic addrec"); + assert(!N.isZero() && "This is not a quadratic addrec"); unsigned BitWidth = LC->getAPInt().getBitWidth(); unsigned NewWidth = BitWidth + 1; @@ -9486,9 +9679,7 @@ ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, // N = Distance (as unsigned) if (StepC->getValue()->isOne() || StepC->getValue()->isMinusOne()) { APInt MaxBECount = getUnsignedRangeMax(applyLoopGuards(Distance, L)); - APInt MaxBECountBase = getUnsignedRangeMax(Distance); - if (MaxBECountBase.ult(MaxBECount)) - MaxBECount = MaxBECountBase; + MaxBECount = APIntOps::umin(MaxBECount, getUnsignedRangeMax(Distance)); // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated, // we end up with a loop whose backedge-taken count is n - 1. Detect this @@ -9521,11 +9712,7 @@ ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, const SCEV *Max = getCouldNotCompute(); if (Exact != getCouldNotCompute()) { APInt MaxInt = getUnsignedRangeMax(applyLoopGuards(Exact, L)); - APInt BaseMaxInt = getUnsignedRangeMax(Exact); - if (BaseMaxInt.ult(MaxInt)) - Max = getConstant(BaseMaxInt); - else - Max = getConstant(MaxInt); + Max = getConstant(APIntOps::umin(MaxInt, getUnsignedRangeMax(Exact))); } return ExitLimit(Exact, Max, false, Predicates); } @@ -9533,9 +9720,12 @@ ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, // Solve the general equation. const SCEV *E = SolveLinEquationWithOverflow(StepC->getAPInt(), getNegativeSCEV(Start), *this); - const SCEV *M = E == getCouldNotCompute() - ? E - : getConstant(getUnsignedRangeMax(E)); + + const SCEV *M = E; + if (E != getCouldNotCompute()) { + APInt MaxWithGuards = getUnsignedRangeMax(applyLoopGuards(E, L)); + M = getConstant(APIntOps::umin(MaxWithGuards, getUnsignedRangeMax(E))); + } return ExitLimit(E, M, false, Predicates); } @@ -9911,23 +10101,23 @@ Optional<bool> ScalarEvolution::evaluatePredicate(ICmpInst::Predicate Pred, bool ScalarEvolution::isKnownPredicateAt(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, - const Instruction *Context) { + const Instruction *CtxI) { // TODO: Analyze guards and assumes from Context's block. return isKnownPredicate(Pred, LHS, RHS) || - isBasicBlockEntryGuardedByCond(Context->getParent(), Pred, LHS, RHS); + isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS); } -Optional<bool> -ScalarEvolution::evaluatePredicateAt(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS, - const Instruction *Context) { +Optional<bool> ScalarEvolution::evaluatePredicateAt(ICmpInst::Predicate Pred, + const SCEV *LHS, + const SCEV *RHS, + const Instruction *CtxI) { Optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS); if (KnownWithoutContext) return KnownWithoutContext; - if (isBasicBlockEntryGuardedByCond(Context->getParent(), Pred, LHS, RHS)) + if (isBasicBlockEntryGuardedByCond(CtxI->getParent(), Pred, LHS, RHS)) return true; - else if (isBasicBlockEntryGuardedByCond(Context->getParent(), + else if (isBasicBlockEntryGuardedByCond(CtxI->getParent(), ICmpInst::getInversePredicate(Pred), LHS, RHS)) return false; @@ -10057,7 +10247,7 @@ ScalarEvolution::getLoopInvariantPredicate(ICmpInst::Predicate Pred, Optional<ScalarEvolution::LoopInvariantPredicate> ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations( ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, - const Instruction *Context, const SCEV *MaxIter) { + const Instruction *CtxI, const SCEV *MaxIter) { // Try to prove the following set of facts: // - The predicate is monotonic in the iteration space. // - If the check does not fail on the 1st iteration: @@ -10111,7 +10301,7 @@ ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations( if (Step == MinusOne) NoOverflowPred = CmpInst::getSwappedPredicate(NoOverflowPred); const SCEV *Start = AR->getStart(); - if (!isKnownPredicateAt(NoOverflowPred, Start, Last, Context)) + if (!isKnownPredicateAt(NoOverflowPred, Start, Last, CtxI)) return None; // Everything is fine. @@ -10448,12 +10638,12 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB, // Try to prove (Pred, LHS, RHS) using isImpliedCond. auto ProveViaCond = [&](const Value *Condition, bool Inverse) { - const Instruction *Context = &BB->front(); - if (isImpliedCond(Pred, LHS, RHS, Condition, Inverse, Context)) + const Instruction *CtxI = &BB->front(); + if (isImpliedCond(Pred, LHS, RHS, Condition, Inverse, CtxI)) return true; if (ProvingStrictComparison) { auto ProofFn = [&](ICmpInst::Predicate P) { - return isImpliedCond(P, LHS, RHS, Condition, Inverse, Context); + return isImpliedCond(P, LHS, RHS, Condition, Inverse, CtxI); }; if (SplitAndProve(ProofFn)) return true; @@ -10525,7 +10715,7 @@ bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Value *FoundCondValue, bool Inverse, - const Instruction *Context) { + const Instruction *CtxI) { // False conditions implies anything. Do not bother analyzing it further. if (FoundCondValue == ConstantInt::getBool(FoundCondValue->getContext(), Inverse)) @@ -10541,12 +10731,12 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const Value *Op0, *Op1; if (match(FoundCondValue, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) { if (!Inverse) - return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, Context) || - isImpliedCond(Pred, LHS, RHS, Op1, Inverse, Context); + return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) || + isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI); } else if (match(FoundCondValue, m_LogicalOr(m_Value(Op0), m_Value(Op1)))) { if (Inverse) - return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, Context) || - isImpliedCond(Pred, LHS, RHS, Op1, Inverse, Context); + return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, CtxI) || + isImpliedCond(Pred, LHS, RHS, Op1, Inverse, CtxI); } const ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue); @@ -10563,14 +10753,14 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); - return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context); + return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, CtxI); } bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, ICmpInst::Predicate FoundPred, const SCEV *FoundLHS, const SCEV *FoundRHS, - const Instruction *Context) { + const Instruction *CtxI) { // Balance the types. if (getTypeSizeInBits(LHS->getType()) < getTypeSizeInBits(FoundLHS->getType())) { @@ -10583,12 +10773,14 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, auto BitWidth = getTypeSizeInBits(NarrowType); const SCEV *MaxValue = getZeroExtendExpr( getConstant(APInt::getMaxValue(BitWidth)), WideType); - if (isKnownPredicate(ICmpInst::ICMP_ULE, FoundLHS, MaxValue) && - isKnownPredicate(ICmpInst::ICMP_ULE, FoundRHS, MaxValue)) { + if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundLHS, + MaxValue) && + isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, FoundRHS, + MaxValue)) { const SCEV *TruncFoundLHS = getTruncateExpr(FoundLHS, NarrowType); const SCEV *TruncFoundRHS = getTruncateExpr(FoundRHS, NarrowType); if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, TruncFoundLHS, - TruncFoundRHS, Context)) + TruncFoundRHS, CtxI)) return true; } } @@ -10615,13 +10807,13 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, } } return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS, - FoundRHS, Context); + FoundRHS, CtxI); } bool ScalarEvolution::isImpliedCondBalancedTypes( ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, ICmpInst::Predicate FoundPred, const SCEV *FoundLHS, const SCEV *FoundRHS, - const Instruction *Context) { + const Instruction *CtxI) { assert(getTypeSizeInBits(LHS->getType()) == getTypeSizeInBits(FoundLHS->getType()) && "Types should be balanced!"); @@ -10647,7 +10839,7 @@ bool ScalarEvolution::isImpliedCondBalancedTypes( // Check whether the found predicate is the same as the desired predicate. if (FoundPred == Pred) - return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context); + return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI); // Check whether swapping the found predicate makes it the same as the // desired predicate. @@ -10663,27 +10855,70 @@ bool ScalarEvolution::isImpliedCondBalancedTypes( // do this if it would break canonical constant/addrec ordering. if (!isa<SCEVConstant>(RHS) && !isa<SCEVAddRecExpr>(LHS)) return isImpliedCondOperands(FoundPred, RHS, LHS, FoundLHS, FoundRHS, - Context); + CtxI); if (!isa<SCEVConstant>(FoundRHS) && !isa<SCEVAddRecExpr>(FoundLHS)) - return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS, Context); + return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS, CtxI); - // Don't try to getNotSCEV pointers. - if (LHS->getType()->isPointerTy() || FoundLHS->getType()->isPointerTy()) - return false; + // There's no clear preference between forms 3. and 4., try both. Avoid + // forming getNotSCEV of pointer values as the resulting subtract is + // not legal. + if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() && + isImpliedCondOperands(FoundPred, getNotSCEV(LHS), getNotSCEV(RHS), + FoundLHS, FoundRHS, CtxI)) + return true; - // There's no clear preference between forms 3. and 4., try both. - return isImpliedCondOperands(FoundPred, getNotSCEV(LHS), getNotSCEV(RHS), - FoundLHS, FoundRHS, Context) || - isImpliedCondOperands(Pred, LHS, RHS, getNotSCEV(FoundLHS), - getNotSCEV(FoundRHS), Context); + if (!FoundLHS->getType()->isPointerTy() && + !FoundRHS->getType()->isPointerTy() && + isImpliedCondOperands(Pred, LHS, RHS, getNotSCEV(FoundLHS), + getNotSCEV(FoundRHS), CtxI)) + return true; + + return false; } - // Unsigned comparison is the same as signed comparison when both the operands - // are non-negative. - if (CmpInst::isUnsigned(FoundPred) && - CmpInst::getSignedPredicate(FoundPred) == Pred && - isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) - return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context); + auto IsSignFlippedPredicate = [](CmpInst::Predicate P1, + CmpInst::Predicate P2) { + assert(P1 != P2 && "Handled earlier!"); + return CmpInst::isRelational(P2) && + P1 == CmpInst::getFlippedSignednessPredicate(P2); + }; + if (IsSignFlippedPredicate(Pred, FoundPred)) { + // Unsigned comparison is the same as signed comparison when both the + // operands are non-negative or negative. + if ((isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) || + (isKnownNegative(FoundLHS) && isKnownNegative(FoundRHS))) + return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI); + // Create local copies that we can freely swap and canonicalize our + // conditions to "le/lt". + ICmpInst::Predicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred; + const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS, + *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS; + if (ICmpInst::isGT(CanonicalPred) || ICmpInst::isGE(CanonicalPred)) { + CanonicalPred = ICmpInst::getSwappedPredicate(CanonicalPred); + CanonicalFoundPred = ICmpInst::getSwappedPredicate(CanonicalFoundPred); + std::swap(CanonicalLHS, CanonicalRHS); + std::swap(CanonicalFoundLHS, CanonicalFoundRHS); + } + assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) && + "Must be!"); + assert((ICmpInst::isLT(CanonicalFoundPred) || + ICmpInst::isLE(CanonicalFoundPred)) && + "Must be!"); + if (ICmpInst::isSigned(CanonicalPred) && isKnownNonNegative(CanonicalRHS)) + // Use implication: + // x <u y && y >=s 0 --> x <s y. + // If we can prove the left part, the right part is also proven. + return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS, + CanonicalRHS, CanonicalFoundLHS, + CanonicalFoundRHS); + if (ICmpInst::isUnsigned(CanonicalPred) && isKnownNegative(CanonicalRHS)) + // Use implication: + // x <s y && y <s 0 --> x <u y. + // If we can prove the left part, the right part is also proven. + return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS, + CanonicalRHS, CanonicalFoundLHS, + CanonicalFoundRHS); + } // Check if we can make progress by sharpening ranges. if (FoundPred == ICmpInst::ICMP_NE && @@ -10721,7 +10956,7 @@ bool ScalarEvolution::isImpliedCondBalancedTypes( // We know V `Pred` SharperMin. If this implies LHS `Pred` // RHS, we're done. if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(SharperMin), - Context)) + CtxI)) return true; LLVM_FALLTHROUGH; @@ -10736,8 +10971,7 @@ bool ScalarEvolution::isImpliedCondBalancedTypes( // // If V `Pred` Min implies LHS `Pred` RHS, we're done. - if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min), - Context)) + if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min), CtxI)) return true; break; @@ -10745,14 +10979,14 @@ bool ScalarEvolution::isImpliedCondBalancedTypes( case ICmpInst::ICMP_SLE: case ICmpInst::ICMP_ULE: if (isImpliedCondOperands(CmpInst::getSwappedPredicate(Pred), RHS, - LHS, V, getConstant(SharperMin), Context)) + LHS, V, getConstant(SharperMin), CtxI)) return true; LLVM_FALLTHROUGH; case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_ULT: if (isImpliedCondOperands(CmpInst::getSwappedPredicate(Pred), RHS, - LHS, V, getConstant(Min), Context)) + LHS, V, getConstant(Min), CtxI)) return true; break; @@ -10766,12 +11000,11 @@ bool ScalarEvolution::isImpliedCondBalancedTypes( // Check whether the actual condition is beyond sufficient. if (FoundPred == ICmpInst::ICMP_EQ) if (ICmpInst::isTrueWhenEqual(Pred)) - if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context)) + if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI)) return true; if (Pred == ICmpInst::ICMP_NE) if (!ICmpInst::isTrueWhenEqual(FoundPred)) - if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS, - Context)) + if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS, CtxI)) return true; // Otherwise assume the worst. @@ -10852,7 +11085,7 @@ Optional<APInt> ScalarEvolution::computeConstantDifference(const SCEV *More, bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart( ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, - const SCEV *FoundLHS, const SCEV *FoundRHS, const Instruction *Context) { + const SCEV *FoundLHS, const SCEV *FoundRHS, const Instruction *CtxI) { // Try to recognize the following pattern: // // FoundRHS = ... @@ -10866,9 +11099,9 @@ bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart( // each iteration of this loop, including the first iteration. Therefore, in // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to // prove the original pred using this fact. - if (!Context) + if (!CtxI) return false; - const BasicBlock *ContextBB = Context->getParent(); + const BasicBlock *ContextBB = CtxI->getParent(); // Make sure AR varies in the context block. if (auto *AR = dyn_cast<SCEVAddRecExpr>(FoundLHS)) { const Loop *L = AR->getLoop(); @@ -11090,7 +11323,7 @@ bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS, const SCEV *FoundRHS, - const Instruction *Context) { + const Instruction *CtxI) { if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS)) return true; @@ -11098,7 +11331,7 @@ bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, return true; if (isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS, - Context)) + CtxI)) return true; return isImpliedCondOperandsHelper(Pred, LHS, RHS, @@ -11534,6 +11767,12 @@ const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start, if (IsSigned && BitWidth == 1) return getZero(Stride->getType()); + // This code has only been closely audited for negative strides in the + // unsigned comparison case, it may be correct for signed comparison, but + // that needs to be established. + assert((!IsSigned || !isKnownNonPositive(Stride)) && + "Stride is expected strictly positive for signed case!"); + // Calculate the maximum backedge count based on the range of values // permitted by Start, End, and Stride. APInt MinStart = @@ -11576,6 +11815,80 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); bool PredicatedIV = false; + auto canAssumeNoSelfWrap = [&](const SCEVAddRecExpr *AR) { + // Can we prove this loop *must* be UB if overflow of IV occurs? + // Reasoning goes as follows: + // * Suppose the IV did self wrap. + // * If Stride evenly divides the iteration space, then once wrap + // occurs, the loop must revisit the same values. + // * We know that RHS is invariant, and that none of those values + // caused this exit to be taken previously. Thus, this exit is + // dynamically dead. + // * If this is the sole exit, then a dead exit implies the loop + // must be infinite if there are no abnormal exits. + // * If the loop were infinite, then it must either not be mustprogress + // or have side effects. Otherwise, it must be UB. + // * It can't (by assumption), be UB so we have contradicted our + // premise and can conclude the IV did not in fact self-wrap. + if (!isLoopInvariant(RHS, L)) + return false; + + auto *StrideC = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this)); + if (!StrideC || !StrideC->getAPInt().isPowerOf2()) + return false; + + if (!ControlsExit || !loopHasNoAbnormalExits(L)) + return false; + + return loopIsFiniteByAssumption(L); + }; + + if (!IV) { + if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS)) { + const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ZExt->getOperand()); + if (AR && AR->getLoop() == L && AR->isAffine()) { + auto canProveNUW = [&]() { + if (!isLoopInvariant(RHS, L)) + return false; + + if (!isKnownNonZero(AR->getStepRecurrence(*this))) + // We need the sequence defined by AR to strictly increase in the + // unsigned integer domain for the logic below to hold. + return false; + + const unsigned InnerBitWidth = getTypeSizeInBits(AR->getType()); + const unsigned OuterBitWidth = getTypeSizeInBits(RHS->getType()); + // If RHS <=u Limit, then there must exist a value V in the sequence + // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and + // V <=u UINT_MAX. Thus, we must exit the loop before unsigned + // overflow occurs. This limit also implies that a signed comparison + // (in the wide bitwidth) is equivalent to an unsigned comparison as + // the high bits on both sides must be zero. + APInt StrideMax = getUnsignedRangeMax(AR->getStepRecurrence(*this)); + APInt Limit = APInt::getMaxValue(InnerBitWidth) - (StrideMax - 1); + Limit = Limit.zext(OuterBitWidth); + return getUnsignedRangeMax(applyLoopGuards(RHS, L)).ule(Limit); + }; + auto Flags = AR->getNoWrapFlags(); + if (!hasFlags(Flags, SCEV::FlagNUW) && canProveNUW()) + Flags = setFlags(Flags, SCEV::FlagNUW); + + setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags); + if (AR->hasNoUnsignedWrap()) { + // Emulate what getZeroExtendExpr would have done during construction + // if we'd been able to infer the fact just above at that time. + const SCEV *Step = AR->getStepRecurrence(*this); + Type *Ty = ZExt->getType(); + auto *S = getAddRecExpr( + getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, 0), + getZeroExtendExpr(Step, Ty, 0), L, AR->getNoWrapFlags()); + IV = dyn_cast<SCEVAddRecExpr>(S); + } + } + } + } + + if (!IV && AllowPredicates) { // Try to make this an AddRec using runtime tests, in the first X // iterations of this loop, where X is the SCEV expression found by the @@ -11626,32 +11939,29 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, // // a) IV is either nuw or nsw depending upon signedness (indicated by the // NoWrap flag). - // b) loop is single exit with no side effects. - // + // b) the loop is guaranteed to be finite (e.g. is mustprogress and has + // no side effects within the loop) + // c) loop has a single static exit (with no abnormal exits) // // Precondition a) implies that if the stride is negative, this is a single // trip loop. The backedge taken count formula reduces to zero in this case. // - // Precondition b) implies that if rhs is invariant in L, then unknown - // stride being zero means the backedge can't be taken without UB. + // Precondition b) and c) combine to imply that if rhs is invariant in L, + // then a zero stride means the backedge can't be taken without executing + // undefined behavior. // // The positive stride case is the same as isKnownPositive(Stride) returning // true (original behavior of the function). // - // We want to make sure that the stride is truly unknown as there are edge - // cases where ScalarEvolution propagates no wrap flags to the - // post-increment/decrement IV even though the increment/decrement operation - // itself is wrapping. The computed backedge taken count may be wrong in - // such cases. This is prevented by checking that the stride is not known to - // be either positive or non-positive. For example, no wrap flags are - // propagated to the post-increment IV of this loop with a trip count of 2 - - // - // unsigned char i; - // for(i=127; i<128; i+=129) - // A[i] = i; - // - if (PredicatedIV || !NoWrap || isKnownNonPositive(Stride) || - !loopIsFiniteByAssumption(L)) + if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) || + !loopHasNoAbnormalExits(L)) + return getCouldNotCompute(); + + // This bailout is protecting the logic in computeMaxBECountForLT which + // has not yet been sufficiently auditted or tested with negative strides. + // We used to filter out all known-non-positive cases here, we're in the + // process of being less restrictive bit by bit. + if (IsSigned && isKnownNonPositive(Stride)) return getCouldNotCompute(); if (!isKnownNonZero(Stride)) { @@ -11687,37 +11997,12 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, } } else if (!Stride->isOne() && !NoWrap) { auto isUBOnWrap = [&]() { - // Can we prove this loop *must* be UB if overflow of IV occurs? - // Reasoning goes as follows: - // * Suppose the IV did self wrap. - // * If Stride evenly divides the iteration space, then once wrap - // occurs, the loop must revisit the same values. - // * We know that RHS is invariant, and that none of those values - // caused this exit to be taken previously. Thus, this exit is - // dynamically dead. - // * If this is the sole exit, then a dead exit implies the loop - // must be infinite if there are no abnormal exits. - // * If the loop were infinite, then it must either not be mustprogress - // or have side effects. Otherwise, it must be UB. - // * It can't (by assumption), be UB so we have contradicted our - // premise and can conclude the IV did not in fact self-wrap. // From no-self-wrap, we need to then prove no-(un)signed-wrap. This // follows trivially from the fact that every (un)signed-wrapped, but // not self-wrapped value must be LT than the last value before // (un)signed wrap. Since we know that last value didn't exit, nor // will any smaller one. - - if (!isLoopInvariant(RHS, L)) - return false; - - auto *StrideC = dyn_cast<SCEVConstant>(Stride); - if (!StrideC || !StrideC->getAPInt().isPowerOf2()) - return false; - - if (!ControlsExit || !loopHasNoAbnormalExits(L)) - return false; - - return loopIsFiniteByAssumption(L); + return canAssumeNoSelfWrap(IV); }; // Avoid proven overflow cases: this will ensure that the backedge taken @@ -11740,7 +12025,9 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, const SCEV *Start = IV->getStart(); // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond. - // Use integer-typed versions for actual computation. + // If we convert to integers, isLoopEntryGuardedByCond will miss some cases. + // Use integer-typed versions for actual computation; we can't subtract + // pointers in general. const SCEV *OrigStart = Start; const SCEV *OrigRHS = RHS; if (Start->getType()->isPointerTy()) { @@ -11771,10 +12058,13 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, // is End and so the result is as above, and if not max(End,Start) is Start // so we get a backedge count of zero. const SCEV *BECount = nullptr; - auto *StartMinusStride = getMinusSCEV(OrigStart, Stride); + auto *OrigStartMinusStride = getMinusSCEV(OrigStart, Stride); + assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!"); + assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!"); + assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!"); // Can we prove (max(RHS,Start) > Start - Stride? - if (isLoopEntryGuardedByCond(L, Cond, StartMinusStride, Start) && - isLoopEntryGuardedByCond(L, Cond, StartMinusStride, RHS)) { + if (isLoopEntryGuardedByCond(L, Cond, OrigStartMinusStride, OrigStart) && + isLoopEntryGuardedByCond(L, Cond, OrigStartMinusStride, OrigRHS)) { // In this case, we can use a refined formula for computing backedge taken // count. The general formula remains: // "End-Start /uceiling Stride" where "End = max(RHS,Start)" @@ -11795,10 +12085,8 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, // Our preconditions trivially imply no overflow in that form. const SCEV *MinusOne = getMinusOne(Stride->getType()); const SCEV *Numerator = - getMinusSCEV(getAddExpr(RHS, MinusOne), StartMinusStride); - if (!isa<SCEVCouldNotCompute>(Numerator)) { - BECount = getUDivExpr(Numerator, Stride); - } + getMinusSCEV(getAddExpr(RHS, MinusOne), getMinusSCEV(Start, Stride)); + BECount = getUDivExpr(Numerator, Stride); } const SCEV *BECountIfBackedgeTaken = nullptr; @@ -12141,7 +12429,7 @@ SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const { } // Return true when S contains at least an undef value. -static inline bool containsUndefs(const SCEV *S) { +bool ScalarEvolution::containsUndefs(const SCEV *S) const { return SCEVExprContains(S, [](const SCEV *S) { if (const auto *SU = dyn_cast<SCEVUnknown>(S)) return isa<UndefValue>(SU->getValue()); @@ -12149,237 +12437,6 @@ static inline bool containsUndefs(const SCEV *S) { }); } -namespace { - -// Collect all steps of SCEV expressions. -struct SCEVCollectStrides { - ScalarEvolution &SE; - SmallVectorImpl<const SCEV *> &Strides; - - SCEVCollectStrides(ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &S) - : SE(SE), Strides(S) {} - - bool follow(const SCEV *S) { - if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) - Strides.push_back(AR->getStepRecurrence(SE)); - return true; - } - - bool isDone() const { return false; } -}; - -// Collect all SCEVUnknown and SCEVMulExpr expressions. -struct SCEVCollectTerms { - SmallVectorImpl<const SCEV *> &Terms; - - SCEVCollectTerms(SmallVectorImpl<const SCEV *> &T) : Terms(T) {} - - bool follow(const SCEV *S) { - if (isa<SCEVUnknown>(S) || isa<SCEVMulExpr>(S) || - isa<SCEVSignExtendExpr>(S)) { - if (!containsUndefs(S)) - Terms.push_back(S); - - // Stop recursion: once we collected a term, do not walk its operands. - return false; - } - - // Keep looking. - return true; - } - - bool isDone() const { return false; } -}; - -// Check if a SCEV contains an AddRecExpr. -struct SCEVHasAddRec { - bool &ContainsAddRec; - - SCEVHasAddRec(bool &ContainsAddRec) : ContainsAddRec(ContainsAddRec) { - ContainsAddRec = false; - } - - bool follow(const SCEV *S) { - if (isa<SCEVAddRecExpr>(S)) { - ContainsAddRec = true; - - // Stop recursion: once we collected a term, do not walk its operands. - return false; - } - - // Keep looking. - return true; - } - - bool isDone() const { return false; } -}; - -// Find factors that are multiplied with an expression that (possibly as a -// subexpression) contains an AddRecExpr. In the expression: -// -// 8 * (100 + %p * %q * (%a + {0, +, 1}_loop)) -// -// "%p * %q" are factors multiplied by the expression "(%a + {0, +, 1}_loop)" -// that contains the AddRec {0, +, 1}_loop. %p * %q are likely to be array size -// parameters as they form a product with an induction variable. -// -// This collector expects all array size parameters to be in the same MulExpr. -// It might be necessary to later add support for collecting parameters that are -// spread over different nested MulExpr. -struct SCEVCollectAddRecMultiplies { - SmallVectorImpl<const SCEV *> &Terms; - ScalarEvolution &SE; - - SCEVCollectAddRecMultiplies(SmallVectorImpl<const SCEV *> &T, ScalarEvolution &SE) - : Terms(T), SE(SE) {} - - bool follow(const SCEV *S) { - if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) { - bool HasAddRec = false; - SmallVector<const SCEV *, 0> Operands; - for (auto Op : Mul->operands()) { - const SCEVUnknown *Unknown = dyn_cast<SCEVUnknown>(Op); - if (Unknown && !isa<CallInst>(Unknown->getValue())) { - Operands.push_back(Op); - } else if (Unknown) { - HasAddRec = true; - } else { - bool ContainsAddRec = false; - SCEVHasAddRec ContiansAddRec(ContainsAddRec); - visitAll(Op, ContiansAddRec); - HasAddRec |= ContainsAddRec; - } - } - if (Operands.size() == 0) - return true; - - if (!HasAddRec) - return false; - - Terms.push_back(SE.getMulExpr(Operands)); - // Stop recursion: once we collected a term, do not walk its operands. - return false; - } - - // Keep looking. - return true; - } - - bool isDone() const { return false; } -}; - -} // end anonymous namespace - -/// Find parametric terms in this SCEVAddRecExpr. We first for parameters in -/// two places: -/// 1) The strides of AddRec expressions. -/// 2) Unknowns that are multiplied with AddRec expressions. -void ScalarEvolution::collectParametricTerms(const SCEV *Expr, - SmallVectorImpl<const SCEV *> &Terms) { - SmallVector<const SCEV *, 4> Strides; - SCEVCollectStrides StrideCollector(*this, Strides); - visitAll(Expr, StrideCollector); - - LLVM_DEBUG({ - dbgs() << "Strides:\n"; - for (const SCEV *S : Strides) - dbgs() << *S << "\n"; - }); - - for (const SCEV *S : Strides) { - SCEVCollectTerms TermCollector(Terms); - visitAll(S, TermCollector); - } - - LLVM_DEBUG({ - dbgs() << "Terms:\n"; - for (const SCEV *T : Terms) - dbgs() << *T << "\n"; - }); - - SCEVCollectAddRecMultiplies MulCollector(Terms, *this); - visitAll(Expr, MulCollector); -} - -static bool findArrayDimensionsRec(ScalarEvolution &SE, - SmallVectorImpl<const SCEV *> &Terms, - SmallVectorImpl<const SCEV *> &Sizes) { - int Last = Terms.size() - 1; - const SCEV *Step = Terms[Last]; - - // End of recursion. - if (Last == 0) { - if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) { - SmallVector<const SCEV *, 2> Qs; - for (const SCEV *Op : M->operands()) - if (!isa<SCEVConstant>(Op)) - Qs.push_back(Op); - - Step = SE.getMulExpr(Qs); - } - - Sizes.push_back(Step); - return true; - } - - for (const SCEV *&Term : Terms) { - // Normalize the terms before the next call to findArrayDimensionsRec. - const SCEV *Q, *R; - SCEVDivision::divide(SE, Term, Step, &Q, &R); - - // Bail out when GCD does not evenly divide one of the terms. - if (!R->isZero()) - return false; - - Term = Q; - } - - // Remove all SCEVConstants. - erase_if(Terms, [](const SCEV *E) { return isa<SCEVConstant>(E); }); - - if (Terms.size() > 0) - if (!findArrayDimensionsRec(SE, Terms, Sizes)) - return false; - - Sizes.push_back(Step); - return true; -} - -// Returns true when one of the SCEVs of Terms contains a SCEVUnknown parameter. -static inline bool containsParameters(SmallVectorImpl<const SCEV *> &Terms) { - for (const SCEV *T : Terms) - if (SCEVExprContains(T, [](const SCEV *S) { return isa<SCEVUnknown>(S); })) - return true; - - return false; -} - -// Return the number of product terms in S. -static inline int numberOfTerms(const SCEV *S) { - if (const SCEVMulExpr *Expr = dyn_cast<SCEVMulExpr>(S)) - return Expr->getNumOperands(); - return 1; -} - -static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) { - if (isa<SCEVConstant>(T)) - return nullptr; - - if (isa<SCEVUnknown>(T)) - return T; - - if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) { - SmallVector<const SCEV *, 2> Factors; - for (const SCEV *Op : M->operands()) - if (!isa<SCEVConstant>(Op)) - Factors.push_back(Op); - - return SE.getMulExpr(Factors); - } - - return T; -} - /// Return the size of an element read or written by Inst. const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) { Type *Ty; @@ -12394,248 +12451,6 @@ const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) { return getSizeOfExpr(ETy, Ty); } -void ScalarEvolution::findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms, - SmallVectorImpl<const SCEV *> &Sizes, - const SCEV *ElementSize) { - if (Terms.size() < 1 || !ElementSize) - return; - - // Early return when Terms do not contain parameters: we do not delinearize - // non parametric SCEVs. - if (!containsParameters(Terms)) - return; - - LLVM_DEBUG({ - dbgs() << "Terms:\n"; - for (const SCEV *T : Terms) - dbgs() << *T << "\n"; - }); - - // Remove duplicates. - array_pod_sort(Terms.begin(), Terms.end()); - Terms.erase(std::unique(Terms.begin(), Terms.end()), Terms.end()); - - // Put larger terms first. - llvm::sort(Terms, [](const SCEV *LHS, const SCEV *RHS) { - return numberOfTerms(LHS) > numberOfTerms(RHS); - }); - - // Try to divide all terms by the element size. If term is not divisible by - // element size, proceed with the original term. - for (const SCEV *&Term : Terms) { - const SCEV *Q, *R; - SCEVDivision::divide(*this, Term, ElementSize, &Q, &R); - if (!Q->isZero()) - Term = Q; - } - - SmallVector<const SCEV *, 4> NewTerms; - - // Remove constant factors. - for (const SCEV *T : Terms) - if (const SCEV *NewT = removeConstantFactors(*this, T)) - NewTerms.push_back(NewT); - - LLVM_DEBUG({ - dbgs() << "Terms after sorting:\n"; - for (const SCEV *T : NewTerms) - dbgs() << *T << "\n"; - }); - - if (NewTerms.empty() || !findArrayDimensionsRec(*this, NewTerms, Sizes)) { - Sizes.clear(); - return; - } - - // The last element to be pushed into Sizes is the size of an element. - Sizes.push_back(ElementSize); - - LLVM_DEBUG({ - dbgs() << "Sizes:\n"; - for (const SCEV *S : Sizes) - dbgs() << *S << "\n"; - }); -} - -void ScalarEvolution::computeAccessFunctions( - const SCEV *Expr, SmallVectorImpl<const SCEV *> &Subscripts, - SmallVectorImpl<const SCEV *> &Sizes) { - // Early exit in case this SCEV is not an affine multivariate function. - if (Sizes.empty()) - return; - - if (auto *AR = dyn_cast<SCEVAddRecExpr>(Expr)) - if (!AR->isAffine()) - return; - - const SCEV *Res = Expr; - int Last = Sizes.size() - 1; - for (int i = Last; i >= 0; i--) { - const SCEV *Q, *R; - SCEVDivision::divide(*this, Res, Sizes[i], &Q, &R); - - LLVM_DEBUG({ - dbgs() << "Res: " << *Res << "\n"; - dbgs() << "Sizes[i]: " << *Sizes[i] << "\n"; - dbgs() << "Res divided by Sizes[i]:\n"; - dbgs() << "Quotient: " << *Q << "\n"; - dbgs() << "Remainder: " << *R << "\n"; - }); - - Res = Q; - - // Do not record the last subscript corresponding to the size of elements in - // the array. - if (i == Last) { - - // Bail out if the remainder is too complex. - if (isa<SCEVAddRecExpr>(R)) { - Subscripts.clear(); - Sizes.clear(); - return; - } - - continue; - } - - // Record the access function for the current subscript. - Subscripts.push_back(R); - } - - // Also push in last position the remainder of the last division: it will be - // the access function of the innermost dimension. - Subscripts.push_back(Res); - - std::reverse(Subscripts.begin(), Subscripts.end()); - - LLVM_DEBUG({ - dbgs() << "Subscripts:\n"; - for (const SCEV *S : Subscripts) - dbgs() << *S << "\n"; - }); -} - -/// Splits the SCEV into two vectors of SCEVs representing the subscripts and -/// sizes of an array access. Returns the remainder of the delinearization that -/// is the offset start of the array. The SCEV->delinearize algorithm computes -/// the multiples of SCEV coefficients: that is a pattern matching of sub -/// expressions in the stride and base of a SCEV corresponding to the -/// computation of a GCD (greatest common divisor) of base and stride. When -/// SCEV->delinearize fails, it returns the SCEV unchanged. -/// -/// For example: when analyzing the memory access A[i][j][k] in this loop nest -/// -/// void foo(long n, long m, long o, double A[n][m][o]) { -/// -/// for (long i = 0; i < n; i++) -/// for (long j = 0; j < m; j++) -/// for (long k = 0; k < o; k++) -/// A[i][j][k] = 1.0; -/// } -/// -/// the delinearization input is the following AddRec SCEV: -/// -/// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k> -/// -/// From this SCEV, we are able to say that the base offset of the access is %A -/// because it appears as an offset that does not divide any of the strides in -/// the loops: -/// -/// CHECK: Base offset: %A -/// -/// and then SCEV->delinearize determines the size of some of the dimensions of -/// the array as these are the multiples by which the strides are happening: -/// -/// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double) bytes. -/// -/// Note that the outermost dimension remains of UnknownSize because there are -/// no strides that would help identifying the size of the last dimension: when -/// the array has been statically allocated, one could compute the size of that -/// dimension by dividing the overall size of the array by the size of the known -/// dimensions: %m * %o * 8. -/// -/// Finally delinearize provides the access functions for the array reference -/// that does correspond to A[i][j][k] of the above C testcase: -/// -/// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>] -/// -/// The testcases are checking the output of a function pass: -/// DelinearizationPass that walks through all loads and stores of a function -/// asking for the SCEV of the memory access with respect to all enclosing -/// loops, calling SCEV->delinearize on that and printing the results. -void ScalarEvolution::delinearize(const SCEV *Expr, - SmallVectorImpl<const SCEV *> &Subscripts, - SmallVectorImpl<const SCEV *> &Sizes, - const SCEV *ElementSize) { - // First step: collect parametric terms. - SmallVector<const SCEV *, 4> Terms; - collectParametricTerms(Expr, Terms); - - if (Terms.empty()) - return; - - // Second step: find subscript sizes. - findArrayDimensions(Terms, Sizes, ElementSize); - - if (Sizes.empty()) - return; - - // Third step: compute the access functions for each subscript. - computeAccessFunctions(Expr, Subscripts, Sizes); - - if (Subscripts.empty()) - return; - - LLVM_DEBUG({ - dbgs() << "succeeded to delinearize " << *Expr << "\n"; - dbgs() << "ArrayDecl[UnknownSize]"; - for (const SCEV *S : Sizes) - dbgs() << "[" << *S << "]"; - - dbgs() << "\nArrayRef"; - for (const SCEV *S : Subscripts) - dbgs() << "[" << *S << "]"; - dbgs() << "\n"; - }); -} - -bool ScalarEvolution::getIndexExpressionsFromGEP( - const GetElementPtrInst *GEP, SmallVectorImpl<const SCEV *> &Subscripts, - SmallVectorImpl<int> &Sizes) { - assert(Subscripts.empty() && Sizes.empty() && - "Expected output lists to be empty on entry to this function."); - assert(GEP && "getIndexExpressionsFromGEP called with a null GEP"); - Type *Ty = nullptr; - bool DroppedFirstDim = false; - for (unsigned i = 1; i < GEP->getNumOperands(); i++) { - const SCEV *Expr = getSCEV(GEP->getOperand(i)); - if (i == 1) { - Ty = GEP->getSourceElementType(); - if (auto *Const = dyn_cast<SCEVConstant>(Expr)) - if (Const->getValue()->isZero()) { - DroppedFirstDim = true; - continue; - } - Subscripts.push_back(Expr); - continue; - } - - auto *ArrayTy = dyn_cast<ArrayType>(Ty); - if (!ArrayTy) { - Subscripts.clear(); - Sizes.clear(); - return false; - } - - Subscripts.push_back(Expr); - if (!(DroppedFirstDim && i == 2)) - Sizes.push_back(ArrayTy->getNumElements()); - - Ty = ArrayTy->getElementType(); - } - return !Subscripts.empty(); -} - //===----------------------------------------------------------------------===// // SCEVCallbackVH Class Implementation //===----------------------------------------------------------------------===// @@ -12722,6 +12537,7 @@ ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg) LoopDispositions(std::move(Arg.LoopDispositions)), LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)), BlockDispositions(std::move(Arg.BlockDispositions)), + SCEVUsers(std::move(Arg.SCEVUsers)), UnsignedRanges(std::move(Arg.UnsignedRanges)), SignedRanges(std::move(Arg.SignedRanges)), UniqueSCEVs(std::move(Arg.UniqueSCEVs)), @@ -12934,7 +12750,7 @@ ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) { Values.emplace_back(L, LoopVariant); LoopDisposition D = computeLoopDisposition(S, L); auto &Values2 = LoopDispositions[S]; - for (auto &V : make_range(Values2.rbegin(), Values2.rend())) { + for (auto &V : llvm::reverse(Values2)) { if (V.getPointer() == L) { V.setInt(D); break; @@ -13042,7 +12858,7 @@ ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { Values.emplace_back(BB, DoesNotDominateBlock); BlockDisposition D = computeBlockDisposition(S, BB); auto &Values2 = BlockDispositions[S]; - for (auto &V : make_range(Values2.rbegin(), Values2.rend())) { + for (auto &V : llvm::reverse(Values2)) { if (V.getPointer() == BB) { V.setInt(D); break; @@ -13130,41 +12946,58 @@ bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { return SCEVExprContains(S, [&](const SCEV *Expr) { return Expr == Op; }); } -void -ScalarEvolution::forgetMemoizedResults(const SCEV *S) { - ValuesAtScopes.erase(S); - LoopDispositions.erase(S); - BlockDispositions.erase(S); - UnsignedRanges.erase(S); - SignedRanges.erase(S); - ExprValueMap.erase(S); - HasRecMap.erase(S); - MinTrailingZerosCache.erase(S); +void ScalarEvolution::forgetMemoizedResults(ArrayRef<const SCEV *> SCEVs) { + SmallPtrSet<const SCEV *, 8> ToForget(SCEVs.begin(), SCEVs.end()); + SmallVector<const SCEV *, 8> Worklist(ToForget.begin(), ToForget.end()); + + while (!Worklist.empty()) { + const SCEV *Curr = Worklist.pop_back_val(); + auto Users = SCEVUsers.find(Curr); + if (Users != SCEVUsers.end()) + for (auto *User : Users->second) + if (ToForget.insert(User).second) + Worklist.push_back(User); + } + + for (auto *S : ToForget) + forgetMemoizedResultsImpl(S); for (auto I = PredicatedSCEVRewrites.begin(); I != PredicatedSCEVRewrites.end();) { std::pair<const SCEV *, const Loop *> Entry = I->first; - if (Entry.first == S) + if (ToForget.count(Entry.first)) PredicatedSCEVRewrites.erase(I++); else ++I; } - auto RemoveSCEVFromBackedgeMap = - [S](DenseMap<const Loop *, BackedgeTakenInfo> &Map) { + auto RemoveSCEVFromBackedgeMap = [&ToForget]( + DenseMap<const Loop *, BackedgeTakenInfo> &Map) { for (auto I = Map.begin(), E = Map.end(); I != E;) { BackedgeTakenInfo &BEInfo = I->second; - if (BEInfo.hasOperand(S)) + if (any_of(ToForget, + [&BEInfo](const SCEV *S) { return BEInfo.hasOperand(S); })) Map.erase(I++); else ++I; } - }; + }; RemoveSCEVFromBackedgeMap(BackedgeTakenCounts); RemoveSCEVFromBackedgeMap(PredicatedBackedgeTakenCounts); } +void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) { + ValuesAtScopes.erase(S); + LoopDispositions.erase(S); + BlockDispositions.erase(S); + UnsignedRanges.erase(S); + SignedRanges.erase(S); + ExprValueMap.erase(S); + HasRecMap.erase(S); + MinTrailingZerosCache.erase(S); +} + void ScalarEvolution::getUsedLoops(const SCEV *S, SmallPtrSetImpl<const Loop *> &LoopsUsed) { @@ -13185,13 +13018,6 @@ ScalarEvolution::getUsedLoops(const SCEV *S, SCEVTraversal<FindUsedLoops>(F).visitAll(S); } -void ScalarEvolution::addToLoopUseLists(const SCEV *S) { - SmallPtrSet<const Loop *, 8> LoopsUsed; - getUsedLoops(S, LoopsUsed); - for (auto *L : LoopsUsed) - LoopUsers[L].push_back(S); -} - void ScalarEvolution::verify() const { ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); ScalarEvolution SE2(F, TLI, AC, DT, LI); @@ -13282,6 +13108,23 @@ void ScalarEvolution::verify() const { assert(ValidLoops.contains(AR->getLoop()) && "AddRec references invalid loop"); } + + // Verify intergity of SCEV users. + for (const auto &S : UniqueSCEVs) { + SmallVector<const SCEV *, 4> Ops; + collectUniqueOps(&S, Ops); + for (const auto *Op : Ops) { + // We do not store dependencies of constants. + if (isa<SCEVConstant>(Op)) + continue; + auto It = SCEVUsers.find(Op); + if (It != SCEVUsers.end() && It->second.count(&S)) + continue; + dbgs() << "Use of operand " << *Op << " by user " << S + << " is not being tracked!\n"; + std::abort(); + } + } } bool ScalarEvolution::invalidate( @@ -13685,6 +13528,16 @@ PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L) : SE(SE), L(L) {} +void ScalarEvolution::registerUser(const SCEV *User, + ArrayRef<const SCEV *> Ops) { + for (auto *Op : Ops) + // We do not expect that forgetting cached data for SCEVConstants will ever + // open any prospects for sharpening or introduce any correctness issues, + // so we don't bother storing their dependencies. + if (!isa<SCEVConstant>(Op)) + SCEVUsers[Op].insert(User); +} + const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) { const SCEV *Expr = SE.getSCEV(V); RewriteEntry &Entry = RewriteMap[Expr]; @@ -13897,52 +13750,51 @@ ScalarEvolution::computeSymbolicMaxBackedgeTakenCount(const Loop *L) { return getUMinFromMismatchedTypes(ExitCounts); } -/// This rewriter is similar to SCEVParameterRewriter (it replaces SCEVUnknown -/// components following the Map (Value -> SCEV)), but skips AddRecExpr because -/// we cannot guarantee that the replacement is loop invariant in the loop of -/// the AddRec. +/// A rewriter to replace SCEV expressions in Map with the corresponding entry +/// in the map. It skips AddRecExpr because we cannot guarantee that the +/// replacement is loop invariant in the loop of the AddRec. +/// +/// At the moment only rewriting SCEVUnknown and SCEVZeroExtendExpr is +/// supported. class SCEVLoopGuardRewriter : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> { - ValueToSCEVMapTy ⤅ + const DenseMap<const SCEV *, const SCEV *> ⤅ public: - SCEVLoopGuardRewriter(ScalarEvolution &SE, ValueToSCEVMapTy &M) + SCEVLoopGuardRewriter(ScalarEvolution &SE, + DenseMap<const SCEV *, const SCEV *> &M) : SCEVRewriteVisitor(SE), Map(M) {} const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; } const SCEV *visitUnknown(const SCEVUnknown *Expr) { - auto I = Map.find(Expr->getValue()); + auto I = Map.find(Expr); if (I == Map.end()) return Expr; return I->second; } + + const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { + auto I = Map.find(Expr); + if (I == Map.end()) + return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr( + Expr); + return I->second; + } }; const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) { + SmallVector<const SCEV *> ExprsToRewrite; auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS, - const SCEV *RHS, ValueToSCEVMapTy &RewriteMap) { - // If we have LHS == 0, check if LHS is computing a property of some unknown - // SCEV %v which we can rewrite %v to express explicitly. - const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS); - if (Predicate == CmpInst::ICMP_EQ && RHSC && - RHSC->getValue()->isNullValue()) { - // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to - // explicitly express that. - const SCEV *URemLHS = nullptr; - const SCEV *URemRHS = nullptr; - if (matchURem(LHS, URemLHS, URemRHS)) { - if (const SCEVUnknown *LHSUnknown = dyn_cast<SCEVUnknown>(URemLHS)) { - Value *V = LHSUnknown->getValue(); - auto Multiple = - getMulExpr(getUDivExpr(URemLHS, URemRHS), URemRHS, - (SCEV::NoWrapFlags)(SCEV::FlagNUW | SCEV::FlagNSW)); - RewriteMap[V] = Multiple; - return; - } - } - } + const SCEV *RHS, + DenseMap<const SCEV *, const SCEV *> + &RewriteMap) { + // WARNING: It is generally unsound to apply any wrap flags to the proposed + // replacement SCEV which isn't directly implied by the structure of that + // SCEV. In particular, using contextual facts to imply flags is *NOT* + // legal. See the scoping rules for flags in the header to understand why. - if (!isa<SCEVUnknown>(LHS) && isa<SCEVUnknown>(RHS)) { + // If LHS is a constant, apply information to the other expression. + if (isa<SCEVConstant>(LHS)) { std::swap(LHS, RHS); Predicate = CmpInst::getSwappedPredicate(Predicate); } @@ -13950,7 +13802,8 @@ const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) { // Check for a condition of the form (-C1 + X < C2). InstCombine will // create this form when combining two checks of the form (X u< C2 + C1) and // (X >=u C1). - auto MatchRangeCheckIdiom = [this, Predicate, LHS, RHS, &RewriteMap]() { + auto MatchRangeCheckIdiom = [this, Predicate, LHS, RHS, &RewriteMap, + &ExprsToRewrite]() { auto *AddExpr = dyn_cast<SCEVAddExpr>(LHS); if (!AddExpr || AddExpr->getNumOperands() != 2) return false; @@ -13968,26 +13821,55 @@ const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) { // Bail out, unless we have a non-wrapping, monotonic range. if (ExactRegion.isWrappedSet() || ExactRegion.isFullSet()) return false; - auto I = RewriteMap.find(LHSUnknown->getValue()); - const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : LHS; - RewriteMap[LHSUnknown->getValue()] = getUMaxExpr( + auto I = RewriteMap.find(LHSUnknown); + const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : LHSUnknown; + RewriteMap[LHSUnknown] = getUMaxExpr( getConstant(ExactRegion.getUnsignedMin()), getUMinExpr(RewrittenLHS, getConstant(ExactRegion.getUnsignedMax()))); + ExprsToRewrite.push_back(LHSUnknown); return true; }; if (MatchRangeCheckIdiom()) return; - // For now, limit to conditions that provide information about unknown - // expressions. RHS also cannot contain add recurrences. - auto *LHSUnknown = dyn_cast<SCEVUnknown>(LHS); - if (!LHSUnknown || containsAddRecurrence(RHS)) + // If we have LHS == 0, check if LHS is computing a property of some unknown + // SCEV %v which we can rewrite %v to express explicitly. + const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS); + if (Predicate == CmpInst::ICMP_EQ && RHSC && + RHSC->getValue()->isNullValue()) { + // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to + // explicitly express that. + const SCEV *URemLHS = nullptr; + const SCEV *URemRHS = nullptr; + if (matchURem(LHS, URemLHS, URemRHS)) { + if (const SCEVUnknown *LHSUnknown = dyn_cast<SCEVUnknown>(URemLHS)) { + auto Multiple = getMulExpr(getUDivExpr(URemLHS, URemRHS), URemRHS); + RewriteMap[LHSUnknown] = Multiple; + ExprsToRewrite.push_back(LHSUnknown); + return; + } + } + } + + // Do not apply information for constants or if RHS contains an AddRec. + if (isa<SCEVConstant>(LHS) || containsAddRecurrence(RHS)) + return; + + // If RHS is SCEVUnknown, make sure the information is applied to it. + if (!isa<SCEVUnknown>(LHS) && isa<SCEVUnknown>(RHS)) { + std::swap(LHS, RHS); + Predicate = CmpInst::getSwappedPredicate(Predicate); + } + + // Limit to expressions that can be rewritten. + if (!isa<SCEVUnknown>(LHS) && !isa<SCEVZeroExtendExpr>(LHS)) return; // Check whether LHS has already been rewritten. In that case we want to // chain further rewrites onto the already rewritten value. - auto I = RewriteMap.find(LHSUnknown->getValue()); + auto I = RewriteMap.find(LHS); const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : LHS; + const SCEV *RewrittenRHS = nullptr; switch (Predicate) { case CmpInst::ICMP_ULT: @@ -14031,14 +13913,17 @@ const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) { break; } - if (RewrittenRHS) - RewriteMap[LHSUnknown->getValue()] = RewrittenRHS; + if (RewrittenRHS) { + RewriteMap[LHS] = RewrittenRHS; + if (LHS == RewrittenLHS) + ExprsToRewrite.push_back(LHS); + } }; // Starting at the loop predecessor, climb up the predecessor chain, as long // as there are predecessors that can be found that have unique successors // leading to the original header. // TODO: share this logic with isLoopEntryGuardedByCond. - ValueToSCEVMapTy RewriteMap; + DenseMap<const SCEV *, const SCEV *> RewriteMap; for (std::pair<const BasicBlock *, const BasicBlock *> Pair( L->getLoopPredecessor(), L->getHeader()); Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { @@ -14088,6 +13973,19 @@ const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) { if (RewriteMap.empty()) return Expr; + + // Now that all rewrite information is collect, rewrite the collected + // expressions with the information in the map. This applies information to + // sub-expressions. + if (ExprsToRewrite.size() > 1) { + for (const SCEV *Expr : ExprsToRewrite) { + const SCEV *RewriteTo = RewriteMap[Expr]; + RewriteMap.erase(Expr); + SCEVLoopGuardRewriter Rewriter(*this, RewriteMap); + RewriteMap.insert({Expr, Rewriter.visit(RewriteTo)}); + } + } + SCEVLoopGuardRewriter Rewriter(*this, RewriteMap); return Rewriter.visit(Expr); } diff --git a/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp index 2262fc9d7913..f4fa159d1ec7 100644 --- a/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp +++ b/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp @@ -23,6 +23,15 @@ #include "llvm/InitializePasses.h" using namespace llvm; +static bool canComputePointerDiff(ScalarEvolution &SE, + const SCEV *A, const SCEV *B) { + if (SE.getEffectiveSCEVType(A->getType()) != + SE.getEffectiveSCEVType(B->getType())) + return false; + + return SE.instructionCouldExistWitthOperands(A, B); +} + AliasResult SCEVAAResult::alias(const MemoryLocation &LocA, const MemoryLocation &LocB, AAQueryInfo &AAQI) { // If either of the memory references is empty, it doesn't matter what the @@ -41,8 +50,7 @@ AliasResult SCEVAAResult::alias(const MemoryLocation &LocA, // If something is known about the difference between the two addresses, // see if it's enough to prove a NoAlias. - if (SE.getEffectiveSCEVType(AS->getType()) == - SE.getEffectiveSCEVType(BS->getType())) { + if (canComputePointerDiff(SE, AS, BS)) { unsigned BitWidth = SE.getTypeSizeInBits(AS->getType()); APInt ASizeInt(BitWidth, LocA.Size.hasValue() ? LocA.Size.getValue() diff --git a/llvm/lib/Analysis/StackLifetime.cpp b/llvm/lib/Analysis/StackLifetime.cpp index ab5f2db7d1cd..9056cc01484d 100644 --- a/llvm/lib/Analysis/StackLifetime.cpp +++ b/llvm/lib/Analysis/StackLifetime.cpp @@ -257,14 +257,12 @@ void StackLifetime::calculateLiveIntervals() { unsigned AllocaNo = It.second.AllocaNo; if (IsStart) { - assert(!Started.test(AllocaNo) || Start[AllocaNo] == BBStart); if (!Started.test(AllocaNo)) { Started.set(AllocaNo); Ended.reset(AllocaNo); Start[AllocaNo] = InstNo; } } else { - assert(!Ended.test(AllocaNo)); if (Started.test(AllocaNo)) { LiveRanges[AllocaNo].addRange(Start[AllocaNo], InstNo); Started.reset(AllocaNo); @@ -400,3 +398,19 @@ PreservedAnalyses StackLifetimePrinterPass::run(Function &F, SL.print(OS); return PreservedAnalyses::all(); } + +void StackLifetimePrinterPass::printPipeline( + raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) { + static_cast<PassInfoMixin<StackLifetimePrinterPass> *>(this)->printPipeline( + OS, MapClassName2PassName); + OS << "<"; + switch (Type) { + case StackLifetime::LivenessType::May: + OS << "may"; + break; + case StackLifetime::LivenessType::Must: + OS << "must"; + break; + } + OS << ">"; +} diff --git a/llvm/lib/Analysis/StackSafetyAnalysis.cpp b/llvm/lib/Analysis/StackSafetyAnalysis.cpp index 76f195fedf31..74cc39b7f2c0 100644 --- a/llvm/lib/Analysis/StackSafetyAnalysis.cpp +++ b/llvm/lib/Analysis/StackSafetyAnalysis.cpp @@ -30,6 +30,7 @@ #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <memory> +#include <tuple> using namespace llvm; @@ -116,6 +117,7 @@ template <typename CalleeTy> struct UseInfo { // Access range if the address (alloca or parameters). // It is allowed to be empty-set when there are no known accesses. ConstantRange Range; + std::map<const Instruction *, ConstantRange> Accesses; // List of calls which pass address as an argument. // Value is offset range of address from base address (alloca or calling @@ -129,6 +131,12 @@ template <typename CalleeTy> struct UseInfo { UseInfo(unsigned PointerSize) : Range{PointerSize, false} {} void updateRange(const ConstantRange &R) { Range = unionNoWrap(Range, R); } + void addRange(const Instruction *I, const ConstantRange &R) { + auto Ins = Accesses.emplace(I, R); + if (!Ins.second) + Ins.first->second = unionNoWrap(Ins.first->second, R); + updateRange(R); + } }; template <typename CalleeTy> @@ -146,7 +154,7 @@ raw_ostream &operator<<(raw_ostream &OS, const UseInfo<CalleeTy> &U) { ConstantRange getStaticAllocaSizeRange(const AllocaInst &AI) { const DataLayout &DL = AI.getModule()->getDataLayout(); TypeSize TS = DL.getTypeAllocSize(AI.getAllocatedType()); - unsigned PointerSize = DL.getMaxPointerSizeInBits(); + unsigned PointerSize = DL.getPointerTypeSizeInBits(AI.getType()); // Fallback to empty range for alloca size. ConstantRange R = ConstantRange::getEmpty(PointerSize); if (TS.isScalable()) @@ -167,7 +175,7 @@ ConstantRange getStaticAllocaSizeRange(const AllocaInst &AI) { if (Overflow) return R; } - R = ConstantRange(APInt::getNullValue(PointerSize), APSize); + R = ConstantRange(APInt::getZero(PointerSize), APSize); assert(!isUnsafe(R)); return R; } @@ -208,7 +216,6 @@ template <typename CalleeTy> struct FunctionInfo { } else { assert(Allocas.empty()); } - O << "\n"; } }; @@ -223,6 +230,7 @@ struct StackSafetyInfo::InfoTy { struct StackSafetyGlobalInfo::InfoTy { GVToSSI Info; SmallPtrSet<const AllocaInst *, 8> SafeAllocas; + std::map<const Instruction *, bool> AccessIsUnsafe; }; namespace { @@ -242,7 +250,7 @@ class StackSafetyLocalAnalysis { ConstantRange getMemIntrinsicAccessRange(const MemIntrinsic *MI, const Use &U, Value *Base); - bool analyzeAllUses(Value *Ptr, UseInfo<GlobalValue> &AS, + void analyzeAllUses(Value *Ptr, UseInfo<GlobalValue> &AS, const StackLifetime &SL); public: @@ -297,8 +305,8 @@ ConstantRange StackSafetyLocalAnalysis::getAccessRange(Value *Addr, Value *Base, APInt APSize(PointerSize, Size.getFixedSize(), true); if (APSize.isNegative()) return UnknownRange; - return getAccessRange( - Addr, Base, ConstantRange(APInt::getNullValue(PointerSize), APSize)); + return getAccessRange(Addr, Base, + ConstantRange(APInt::getZero(PointerSize), APSize)); } ConstantRange StackSafetyLocalAnalysis::getMemIntrinsicAccessRange( @@ -321,14 +329,13 @@ ConstantRange StackSafetyLocalAnalysis::getMemIntrinsicAccessRange( if (Sizes.getUpper().isNegative() || isUnsafe(Sizes)) return UnknownRange; Sizes = Sizes.sextOrTrunc(PointerSize); - ConstantRange SizeRange(APInt::getNullValue(PointerSize), - Sizes.getUpper() - 1); + ConstantRange SizeRange(APInt::getZero(PointerSize), Sizes.getUpper() - 1); return getAccessRange(U, Base, SizeRange); } /// The function analyzes all local uses of Ptr (alloca or argument) and /// calculates local access range and all function calls where it was used. -bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, +void StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, UseInfo<GlobalValue> &US, const StackLifetime &SL) { SmallPtrSet<const Value *, 16> Visited; @@ -349,11 +356,11 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, switch (I->getOpcode()) { case Instruction::Load: { if (AI && !SL.isAliveAfter(AI, I)) { - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; } - US.updateRange( - getAccessRange(UI, Ptr, DL.getTypeStoreSize(I->getType()))); + US.addRange(I, + getAccessRange(UI, Ptr, DL.getTypeStoreSize(I->getType()))); break; } @@ -363,15 +370,16 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, case Instruction::Store: { if (V == I->getOperand(0)) { // Stored the pointer - conservatively assume it may be unsafe. - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; } if (AI && !SL.isAliveAfter(AI, I)) { - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; } - US.updateRange(getAccessRange( - UI, Ptr, DL.getTypeStoreSize(I->getOperand(0)->getType()))); + US.addRange( + I, getAccessRange( + UI, Ptr, DL.getTypeStoreSize(I->getOperand(0)->getType()))); break; } @@ -379,8 +387,8 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, // Information leak. // FIXME: Process parameters correctly. This is a leak only if we return // alloca. - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; case Instruction::Call: case Instruction::Invoke: { @@ -388,25 +396,31 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, break; if (AI && !SL.isAliveAfter(AI, I)) { - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; } if (const MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) { - US.updateRange(getMemIntrinsicAccessRange(MI, UI, Ptr)); + US.addRange(I, getMemIntrinsicAccessRange(MI, UI, Ptr)); break; } const auto &CB = cast<CallBase>(*I); + if (CB.getReturnedArgOperand() == V) { + if (Visited.insert(I).second) + WorkList.push_back(cast<const Instruction>(I)); + } + if (!CB.isArgOperand(&UI)) { - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; } unsigned ArgNo = CB.getArgOperandNo(&UI); if (CB.isByValArgument(ArgNo)) { - US.updateRange(getAccessRange( - UI, Ptr, DL.getTypeStoreSize(CB.getParamByValType(ArgNo)))); + US.addRange(I, getAccessRange( + UI, Ptr, + DL.getTypeStoreSize(CB.getParamByValType(ArgNo)))); break; } @@ -416,8 +430,8 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, const GlobalValue *Callee = dyn_cast<GlobalValue>(CB.getCalledOperand()->stripPointerCasts()); if (!Callee) { - US.updateRange(UnknownRange); - return false; + US.addRange(I, UnknownRange); + break; } assert(isa<Function>(Callee) || isa<GlobalAlias>(Callee)); @@ -435,8 +449,6 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(Value *Ptr, } } } - - return true; } FunctionInfo<GlobalValue> StackSafetyLocalAnalysis::run() { @@ -468,7 +480,7 @@ FunctionInfo<GlobalValue> StackSafetyLocalAnalysis::run() { } LLVM_DEBUG(Info.print(dbgs(), F.getName(), &F)); - LLVM_DEBUG(dbgs() << "[StackSafety] done\n"); + LLVM_DEBUG(dbgs() << "\n[StackSafety] done\n"); return Info; } @@ -588,8 +600,7 @@ void StackSafetyDataFlowAnalysis<CalleeTy>::runDataFlow() { updateAllNodes(); while (!WorkList.empty()) { - const CalleeTy *Callee = WorkList.back(); - WorkList.pop_back(); + const CalleeTy *Callee = WorkList.pop_back_val(); updateOneNode(Callee); } } @@ -674,7 +685,7 @@ const Function *findCalleeInModule(const GlobalValue *GV) { const GlobalAlias *A = dyn_cast<GlobalAlias>(GV); if (!A) return nullptr; - GV = A->getBaseObject(); + GV = A->getAliaseeObject(); if (GV == A) return nullptr; } @@ -741,10 +752,8 @@ GVToSSI createGlobalStackSafetyInfo( KV.second.Calls.clear(); } - uint32_t PointerSize = Copy.begin() - ->first->getParent() - ->getDataLayout() - .getMaxPointerSizeInBits(); + uint32_t PointerSize = + Copy.begin()->first->getParent()->getDataLayout().getPointerSizeInBits(); StackSafetyDataFlowAnalysis<GlobalValue> SSDFA(PointerSize, std::move(Copy)); for (auto &F : SSDFA.run()) { @@ -794,6 +803,7 @@ const StackSafetyInfo::InfoTy &StackSafetyInfo::getInfo() const { void StackSafetyInfo::print(raw_ostream &O) const { getInfo().Info.print(O, F->getName(), dyn_cast<Function>(F)); + O << "\n"; } const StackSafetyGlobalInfo::InfoTy &StackSafetyGlobalInfo::getInfo() const { @@ -806,17 +816,22 @@ const StackSafetyGlobalInfo::InfoTy &StackSafetyGlobalInfo::getInfo() const { } } Info.reset(new InfoTy{ - createGlobalStackSafetyInfo(std::move(Functions), Index), {}}); + createGlobalStackSafetyInfo(std::move(Functions), Index), {}, {}}); + for (auto &FnKV : Info->Info) { for (auto &KV : FnKV.second.Allocas) { ++NumAllocaTotal; const AllocaInst *AI = KV.first; - if (getStaticAllocaSizeRange(*AI).contains(KV.second.Range)) { + auto AIRange = getStaticAllocaSizeRange(*AI); + if (AIRange.contains(KV.second.Range)) { Info->SafeAllocas.insert(AI); ++NumAllocaStackSafe; } + for (const auto &A : KV.second.Accesses) + Info->AccessIsUnsafe[A.first] |= !AIRange.contains(A.second); } } + if (StackSafetyPrint) print(errs()); } @@ -886,6 +901,15 @@ bool StackSafetyGlobalInfo::isSafe(const AllocaInst &AI) const { return Info.SafeAllocas.count(&AI); } +bool StackSafetyGlobalInfo::stackAccessIsSafe(const Instruction &I) const { + const auto &Info = getInfo(); + auto It = Info.AccessIsUnsafe.find(&I); + if (It == Info.AccessIsUnsafe.end()) { + return true; + } + return !It->second; +} + void StackSafetyGlobalInfo::print(raw_ostream &O) const { auto &SSI = getInfo().Info; if (SSI.empty()) @@ -894,6 +918,16 @@ void StackSafetyGlobalInfo::print(raw_ostream &O) const { for (auto &F : M.functions()) { if (!F.isDeclaration()) { SSI.find(&F)->second.print(O, F.getName(), &F); + O << " safe accesses:" + << "\n"; + for (const auto &I : instructions(F)) { + const CallInst *Call = dyn_cast<CallInst>(&I); + if ((isa<StoreInst>(I) || isa<LoadInst>(I) || isa<MemIntrinsic>(I) || + (Call && Call->hasByValArgument())) && + stackAccessIsSafe(I)) { + O << " " << I << "\n"; + } + } O << "\n"; } } diff --git a/llvm/lib/Analysis/TFUtils.cpp b/llvm/lib/Analysis/TFUtils.cpp index e93dc303ae63..3d10479c4544 100644 --- a/llvm/lib/Analysis/TFUtils.cpp +++ b/llvm/lib/Analysis/TFUtils.cpp @@ -1,9 +1,8 @@ //===- TFUtils.cpp - tensorflow evaluation utilities ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -262,29 +261,58 @@ private: class LoggerDataImpl { const std::vector<LoggedFeatureSpec> LoggedFeatureSpecs; const TensorSpec RewardSpec; + const bool IncludeReward; - tensorflow::SequenceExample SE; - std::vector<tensorflow::FeatureList *> FeatureLists; - tensorflow::FeatureList *Reward = nullptr; + std::vector<tensorflow::FeatureList> FeatureLists; + tensorflow::FeatureList Reward; -public: - LoggerDataImpl(const std::vector<LoggedFeatureSpec> &LoggedSpecs, - const TensorSpec &RewardSpec, bool IncludeReward) - : LoggedFeatureSpecs(LoggedSpecs), RewardSpec(RewardSpec) { + bool isSelfConsistent(const tensorflow::SequenceExample &SE, + size_t NrRecords) const { + bool Ret = true; + for (const auto &TSpecs : LoggedFeatureSpecs) { + const auto &Name = TSpecs.getLoggingName(); + const auto &FL = SE.feature_lists().feature_list().at(Name).feature(); + if (NrRecords != static_cast<size_t>(FL.size())) { + dbgs() << "[TF-UTILS]: " << Name << " has missing records. Expected " + << NrRecords << " got " << FL.size() << "\n"; + Ret = false; + } + } + if (IncludeReward && static_cast<size_t>(SE.feature_lists() + .feature_list() + .at(RewardSpec.name()) + .feature() + .size()) != NrRecords) { + dbgs() << "[TF-UTILS]: reward is missing records.\n"; + Ret = false; + } + return Ret; + } + + void transferLog(tensorflow::SequenceExample &SE) { auto *FL = SE.mutable_feature_lists()->mutable_feature_list(); if (IncludeReward) - Reward = &(*FL)[RewardSpec.name()]; - // Allocate first the map entries, then capture their address. We will not - // mutate the set of features after this (i.e. the pointers won't dangle). - for (const auto &LFS : LoggedSpecs) { - (*FL)[LFS.LoggingName ? *LFS.LoggingName : LFS.Spec.name()] = {}; + (*FL)[RewardSpec.name()] = std::move(Reward); + assert(FeatureLists.size() == LoggedFeatureSpecs.size()); + for (size_t I = 0; I < FeatureLists.size(); ++I) { + const auto &LFS = LoggedFeatureSpecs[I]; + (*FL)[LFS.getLoggingName()] = std::move(FeatureLists[I]); } - for (const auto &LFS : LoggedSpecs) - FeatureLists.push_back( - &(*FL)[LFS.LoggingName ? *LFS.LoggingName : LFS.Spec.name()]); } - void print(raw_ostream &OS) { +public: + LoggerDataImpl(const std::vector<LoggedFeatureSpec> &LoggedSpecs, + const TensorSpec &RewardSpec, bool IncludeReward) + : LoggedFeatureSpecs(LoggedSpecs), RewardSpec(RewardSpec), + IncludeReward(IncludeReward), FeatureLists(LoggedFeatureSpecs.size()) {} + + // flush the logged info to a stream and clear the log contents. + void flush(raw_ostream &OS) { + size_t NrRecords = getNrRecords(); + (void)NrRecords; + tensorflow::SequenceExample SE; + transferLog(SE); + assert(isSelfConsistent(SE, NrRecords)); std::string OutStr; if (ProtobufTextMode) google::protobuf::TextFormat::PrintToString(SE, &OutStr); @@ -298,14 +326,14 @@ public: const auto &Spec = LoggedFeatureSpecs[FeatureID].Spec; if (Spec.isElementType<float>()) { auto *RF = FeatureLists[FeatureID] - ->add_feature() + .add_feature() ->mutable_float_list() ->mutable_value(); RF->Resize(Spec.getElementCount(), 0.0); return reinterpret_cast<char *>(RF->mutable_data()); } else if (Spec.isElementType<int32_t>() || Spec.isElementType<int64_t>()) { auto *RF = FeatureLists[FeatureID] - ->add_feature() + .add_feature() ->mutable_int64_list() ->mutable_value(); RF->Resize(Spec.getElementCount(), 0); @@ -315,17 +343,18 @@ public: } template <typename T> void logReward(T Value) { + assert(IncludeReward); if (RewardSpec.isElementType<float>()) - Reward->add_feature()->mutable_float_list()->add_value(Value); + Reward.add_feature()->mutable_float_list()->add_value(Value); else if (RewardSpec.isElementType<int32_t>() || RewardSpec.isElementType<int64_t>()) - Reward->add_feature()->mutable_int64_list()->add_value(Value); + Reward.add_feature()->mutable_int64_list()->add_value(Value); else llvm_unreachable("Unsupported tensor type."); } size_t getNrRecords() const { - return FeatureLists.empty() ? 0 : FeatureLists[0]->feature().size(); + return FeatureLists.empty() ? 0 : FeatureLists[0].feature().size(); } }; } // namespace llvm @@ -538,5 +567,5 @@ char *Logger::addEntryAndGetFloatOrInt64Buffer(size_t FeatureID) { return reinterpret_cast<char *>(LoggerData->addNewTensor(FeatureID)); } -void Logger::print(raw_ostream &OS) { LoggerData->print(OS); } +void Logger::flush(raw_ostream &OS) { LoggerData->flush(OS); } #endif // defined(LLVM_HAVE_TF_API) diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp index 4a8818f2e2a8..7326ba74c071 100644 --- a/llvm/lib/Analysis/TargetLibraryInfo.cpp +++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -123,6 +123,7 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, // Set IO unlocked variants as unavailable // Set them as available per system below + TLI.setUnavailable(LibFunc_getc_unlocked); TLI.setUnavailable(LibFunc_getchar_unlocked); TLI.setUnavailable(LibFunc_putc_unlocked); TLI.setUnavailable(LibFunc_putchar_unlocked); @@ -156,15 +157,10 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, // isn't true for a target those defaults should be overridden below. TLI.setIntSize(T.isArch16Bit() ? 16 : 32); - if (T.isAMDGPU()) - TLI.disableAllFunctions(); - - // There are no library implementations of memcpy and memset for AMD gpus and - // these can be difficult to lower in the backend. + // There is really no runtime library on AMDGPU, apart from + // __kmpc_alloc/free_shared. if (T.isAMDGPU()) { - TLI.setUnavailable(LibFunc_memcpy); - TLI.setUnavailable(LibFunc_memset); - TLI.setUnavailable(LibFunc_memset_pattern16); + TLI.disableAllFunctions(); TLI.setAvailable(llvm::LibFunc___kmpc_alloc_shared); TLI.setAvailable(llvm::LibFunc___kmpc_free_shared); return; @@ -418,6 +414,65 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setUnavailable(LibFunc_utimes); } + // Pick just one set of new/delete variants. + if (T.isOSMSVCRT()) { + // MSVC, doesn't have the Itanium new/delete. + TLI.setUnavailable(LibFunc_ZdaPv); + TLI.setUnavailable(LibFunc_ZdaPvRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZdaPvSt11align_val_t); + TLI.setUnavailable(LibFunc_ZdaPvSt11align_val_tRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZdaPvj); + TLI.setUnavailable(LibFunc_ZdaPvjSt11align_val_t); + TLI.setUnavailable(LibFunc_ZdaPvm); + TLI.setUnavailable(LibFunc_ZdaPvmSt11align_val_t); + TLI.setUnavailable(LibFunc_ZdlPv); + TLI.setUnavailable(LibFunc_ZdlPvRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZdlPvSt11align_val_t); + TLI.setUnavailable(LibFunc_ZdlPvSt11align_val_tRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZdlPvj); + TLI.setUnavailable(LibFunc_ZdlPvjSt11align_val_t); + TLI.setUnavailable(LibFunc_ZdlPvm); + TLI.setUnavailable(LibFunc_ZdlPvmSt11align_val_t); + TLI.setUnavailable(LibFunc_Znaj); + TLI.setUnavailable(LibFunc_ZnajRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZnajSt11align_val_t); + TLI.setUnavailable(LibFunc_ZnajSt11align_val_tRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_Znam); + TLI.setUnavailable(LibFunc_ZnamRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZnamSt11align_val_t); + TLI.setUnavailable(LibFunc_ZnamSt11align_val_tRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_Znwj); + TLI.setUnavailable(LibFunc_ZnwjRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZnwjSt11align_val_t); + TLI.setUnavailable(LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_Znwm); + TLI.setUnavailable(LibFunc_ZnwmRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZnwmSt11align_val_t); + TLI.setUnavailable(LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t); + } else { + // Not MSVC, assume it's Itanium. + TLI.setUnavailable(LibFunc_msvc_new_int); + TLI.setUnavailable(LibFunc_msvc_new_int_nothrow); + TLI.setUnavailable(LibFunc_msvc_new_longlong); + TLI.setUnavailable(LibFunc_msvc_new_longlong_nothrow); + TLI.setUnavailable(LibFunc_msvc_delete_ptr32); + TLI.setUnavailable(LibFunc_msvc_delete_ptr32_nothrow); + TLI.setUnavailable(LibFunc_msvc_delete_ptr32_int); + TLI.setUnavailable(LibFunc_msvc_delete_ptr64); + TLI.setUnavailable(LibFunc_msvc_delete_ptr64_nothrow); + TLI.setUnavailable(LibFunc_msvc_delete_ptr64_longlong); + TLI.setUnavailable(LibFunc_msvc_new_array_int); + TLI.setUnavailable(LibFunc_msvc_new_array_int_nothrow); + TLI.setUnavailable(LibFunc_msvc_new_array_longlong); + TLI.setUnavailable(LibFunc_msvc_new_array_longlong_nothrow); + TLI.setUnavailable(LibFunc_msvc_delete_array_ptr32); + TLI.setUnavailable(LibFunc_msvc_delete_array_ptr32_nothrow); + TLI.setUnavailable(LibFunc_msvc_delete_array_ptr32_int); + TLI.setUnavailable(LibFunc_msvc_delete_array_ptr64); + TLI.setUnavailable(LibFunc_msvc_delete_array_ptr64_nothrow); + TLI.setUnavailable(LibFunc_msvc_delete_array_ptr64_longlong); + } + switch (T.getOS()) { case Triple::MacOSX: // exp10 and exp10f are not available on OS X until 10.9 and iOS until 7.0 @@ -572,6 +627,9 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setUnavailable(LibFunc_sinh_finite); TLI.setUnavailable(LibFunc_sinhf_finite); TLI.setUnavailable(LibFunc_sinhl_finite); + TLI.setUnavailable(LibFunc_sqrt_finite); + TLI.setUnavailable(LibFunc_sqrtf_finite); + TLI.setUnavailable(LibFunc_sqrtl_finite); } if ((T.isOSLinux() && T.isGNUEnvironment()) || @@ -589,6 +647,140 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setAvailable(LibFunc_fgets_unlocked); } + if (T.isAndroid() && T.isAndroidVersionLT(21)) { + TLI.setUnavailable(LibFunc_stpcpy); + TLI.setUnavailable(LibFunc_stpncpy); + } + + if (T.isPS4()) { + // PS4 does have memalign. + TLI.setAvailable(LibFunc_memalign); + + // PS4 does not have new/delete with "unsigned int" size parameter; + // it only has the "unsigned long" versions. + TLI.setUnavailable(LibFunc_ZdaPvj); + TLI.setUnavailable(LibFunc_ZdaPvjSt11align_val_t); + TLI.setUnavailable(LibFunc_ZdlPvj); + TLI.setUnavailable(LibFunc_ZdlPvjSt11align_val_t); + TLI.setUnavailable(LibFunc_Znaj); + TLI.setUnavailable(LibFunc_ZnajRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZnajSt11align_val_t); + TLI.setUnavailable(LibFunc_ZnajSt11align_val_tRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_Znwj); + TLI.setUnavailable(LibFunc_ZnwjRKSt9nothrow_t); + TLI.setUnavailable(LibFunc_ZnwjSt11align_val_t); + TLI.setUnavailable(LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t); + + // None of the *_chk functions. + TLI.setUnavailable(LibFunc_memccpy_chk); + TLI.setUnavailable(LibFunc_memcpy_chk); + TLI.setUnavailable(LibFunc_memmove_chk); + TLI.setUnavailable(LibFunc_mempcpy_chk); + TLI.setUnavailable(LibFunc_memset_chk); + TLI.setUnavailable(LibFunc_snprintf_chk); + TLI.setUnavailable(LibFunc_sprintf_chk); + TLI.setUnavailable(LibFunc_stpcpy_chk); + TLI.setUnavailable(LibFunc_stpncpy_chk); + TLI.setUnavailable(LibFunc_strcat_chk); + TLI.setUnavailable(LibFunc_strcpy_chk); + TLI.setUnavailable(LibFunc_strlcat_chk); + TLI.setUnavailable(LibFunc_strlcat_chk); + TLI.setUnavailable(LibFunc_strlcpy_chk); + TLI.setUnavailable(LibFunc_strlen_chk); + TLI.setUnavailable(LibFunc_strncat_chk); + TLI.setUnavailable(LibFunc_strncpy_chk); + TLI.setUnavailable(LibFunc_vsnprintf_chk); + TLI.setUnavailable(LibFunc_vsprintf_chk); + + // Various Posix system functions. + TLI.setUnavailable(LibFunc_access); + TLI.setUnavailable(LibFunc_chmod); + TLI.setUnavailable(LibFunc_chown); + TLI.setUnavailable(LibFunc_closedir); + TLI.setUnavailable(LibFunc_ctermid); + TLI.setUnavailable(LibFunc_execl); + TLI.setUnavailable(LibFunc_execle); + TLI.setUnavailable(LibFunc_execlp); + TLI.setUnavailable(LibFunc_execv); + TLI.setUnavailable(LibFunc_execvP); + TLI.setUnavailable(LibFunc_execve); + TLI.setUnavailable(LibFunc_execvp); + TLI.setUnavailable(LibFunc_execvpe); + TLI.setUnavailable(LibFunc_fork); + TLI.setUnavailable(LibFunc_fstat); + TLI.setUnavailable(LibFunc_fstatvfs); + TLI.setUnavailable(LibFunc_getenv); + TLI.setUnavailable(LibFunc_getitimer); + TLI.setUnavailable(LibFunc_getlogin_r); + TLI.setUnavailable(LibFunc_getpwnam); + TLI.setUnavailable(LibFunc_gettimeofday); + TLI.setUnavailable(LibFunc_lchown); + TLI.setUnavailable(LibFunc_lstat); + TLI.setUnavailable(LibFunc_mkdir); + TLI.setUnavailable(LibFunc_open); + TLI.setUnavailable(LibFunc_opendir); + TLI.setUnavailable(LibFunc_pclose); + TLI.setUnavailable(LibFunc_popen); + TLI.setUnavailable(LibFunc_pread); + TLI.setUnavailable(LibFunc_pwrite); + TLI.setUnavailable(LibFunc_read); + TLI.setUnavailable(LibFunc_readlink); + TLI.setUnavailable(LibFunc_realpath); + TLI.setUnavailable(LibFunc_rename); + TLI.setUnavailable(LibFunc_rmdir); + TLI.setUnavailable(LibFunc_setitimer); + TLI.setUnavailable(LibFunc_stat); + TLI.setUnavailable(LibFunc_statvfs); + TLI.setUnavailable(LibFunc_system); + TLI.setUnavailable(LibFunc_times); + TLI.setUnavailable(LibFunc_tmpfile); + TLI.setUnavailable(LibFunc_unlink); + TLI.setUnavailable(LibFunc_uname); + TLI.setUnavailable(LibFunc_unsetenv); + TLI.setUnavailable(LibFunc_utime); + TLI.setUnavailable(LibFunc_utimes); + TLI.setUnavailable(LibFunc_valloc); + TLI.setUnavailable(LibFunc_write); + + // Miscellaneous other functions not provided. + TLI.setUnavailable(LibFunc_atomic_load); + TLI.setUnavailable(LibFunc_atomic_store); + TLI.setUnavailable(LibFunc___kmpc_alloc_shared); + TLI.setUnavailable(LibFunc___kmpc_free_shared); + TLI.setUnavailable(LibFunc_dunder_strndup); + TLI.setUnavailable(LibFunc_bcmp); + TLI.setUnavailable(LibFunc_bcopy); + TLI.setUnavailable(LibFunc_bzero); + TLI.setUnavailable(LibFunc_cabs); + TLI.setUnavailable(LibFunc_cabsf); + TLI.setUnavailable(LibFunc_cabsl); + TLI.setUnavailable(LibFunc_ffs); + TLI.setUnavailable(LibFunc_flockfile); + TLI.setUnavailable(LibFunc_fseeko); + TLI.setUnavailable(LibFunc_ftello); + TLI.setUnavailable(LibFunc_ftrylockfile); + TLI.setUnavailable(LibFunc_funlockfile); + TLI.setUnavailable(LibFunc_htonl); + TLI.setUnavailable(LibFunc_htons); + TLI.setUnavailable(LibFunc_isascii); + TLI.setUnavailable(LibFunc_memccpy); + TLI.setUnavailable(LibFunc_mempcpy); + TLI.setUnavailable(LibFunc_memrchr); + TLI.setUnavailable(LibFunc_ntohl); + TLI.setUnavailable(LibFunc_ntohs); + TLI.setUnavailable(LibFunc_reallocf); + TLI.setUnavailable(LibFunc_roundeven); + TLI.setUnavailable(LibFunc_roundevenf); + TLI.setUnavailable(LibFunc_roundevenl); + TLI.setUnavailable(LibFunc_stpcpy); + TLI.setUnavailable(LibFunc_stpncpy); + TLI.setUnavailable(LibFunc_strlcat); + TLI.setUnavailable(LibFunc_strlcpy); + TLI.setUnavailable(LibFunc_strndup); + TLI.setUnavailable(LibFunc_strnlen); + TLI.setUnavailable(LibFunc_toascii); + } + // As currently implemented in clang, NVPTX code has no standard library to // speak of. Headers provide a standard-ish library implementation, but many // of the signatures are wrong -- for example, many libm functions are not @@ -691,7 +883,7 @@ TargetLibraryInfoImpl &TargetLibraryInfoImpl::operator=(TargetLibraryInfoImpl && static StringRef sanitizeFunctionName(StringRef funcName) { // Filter out empty names and names containing null bytes, those can't be in // our table. - if (funcName.empty() || funcName.find('\0') != StringRef::npos) + if (funcName.empty() || funcName.contains('\0')) return StringRef(); // Check for \01 prefix that is used to mangle __asm declarations and @@ -716,12 +908,12 @@ bool TargetLibraryInfoImpl::getLibFunc(StringRef funcName, LibFunc &F) const { bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, LibFunc F, - const DataLayout *DL) const { - LLVMContext &Ctx = FTy.getContext(); - Type *SizeTTy = DL ? DL->getIntPtrType(Ctx, /*AddressSpace=*/0) : nullptr; - auto IsSizeTTy = [SizeTTy](Type *Ty) { - return SizeTTy ? Ty == SizeTTy : Ty->isIntegerTy(); - }; + const Module &M) const { + // FIXME: There is really no guarantee that sizeof(size_t) is equal to + // sizeof(int*) for every target. So the assumption used here to derive the + // SizeTBits based on the size of an integer pointer in address space zero + // isn't always valid. + unsigned SizeTBits = M.getDataLayout().getPointerSizeInBits(/*AddrSpace=*/0); unsigned NumParams = FTy.getNumParams(); switch (F) { @@ -745,12 +937,12 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, FTy.getReturnType()->isIntegerTy(32)); case LibFunc_strlen_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_strlen: - return (NumParams == 1 && FTy.getParamType(0)->isPointerTy() && - FTy.getReturnType()->isIntegerTy()); + return NumParams == 1 && FTy.getParamType(0)->isPointerTy() && + FTy.getReturnType()->isIntegerTy(SizeTBits); case LibFunc_strchr: case LibFunc_strrchr: @@ -770,7 +962,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, FTy.getParamType(1)->isPointerTy()); case LibFunc_strcat_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_strcat: @@ -780,19 +972,19 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_strncat_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_strncat: return (NumParams == 3 && FTy.getReturnType()->isPointerTy() && FTy.getParamType(0) == FTy.getReturnType() && FTy.getParamType(1) == FTy.getReturnType() && - IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_strcpy_chk: case LibFunc_stpcpy_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_strcpy: @@ -804,20 +996,20 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_strlcat_chk: case LibFunc_strlcpy_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_strlcat: case LibFunc_strlcpy: - return NumParams == 3 && IsSizeTTy(FTy.getReturnType()) && + return NumParams == 3 && FTy.getReturnType()->isIntegerTy(SizeTBits) && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isPointerTy() && - IsSizeTTy(FTy.getParamType(2)); + FTy.getParamType(2)->isIntegerTy(SizeTBits); case LibFunc_strncpy_chk: case LibFunc_stpncpy_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_strncpy: @@ -825,7 +1017,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, return (NumParams == 3 && FTy.getReturnType() == FTy.getParamType(0) && FTy.getParamType(0) == FTy.getParamType(1) && FTy.getParamType(0)->isPointerTy() && - IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_strxfrm: return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() && @@ -840,7 +1032,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, return (NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(0) == FTy.getParamType(1) && - IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_strspn: case LibFunc_strcspn: @@ -888,20 +1080,21 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_sprintf_chk: return NumParams == 4 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isIntegerTy(32) && - IsSizeTTy(FTy.getParamType(2)) && + FTy.getParamType(2)->isIntegerTy(SizeTBits) && FTy.getParamType(3)->isPointerTy() && FTy.getReturnType()->isIntegerTy(32); case LibFunc_snprintf: - return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() && - FTy.getParamType(2)->isPointerTy() && - FTy.getReturnType()->isIntegerTy(32)); + return NumParams == 3 && FTy.getParamType(0)->isPointerTy() && + FTy.getParamType(1)->isIntegerTy(SizeTBits) && + FTy.getParamType(2)->isPointerTy() && + FTy.getReturnType()->isIntegerTy(32); case LibFunc_snprintf_chk: return NumParams == 5 && FTy.getParamType(0)->isPointerTy() && - IsSizeTTy(FTy.getParamType(1)) && + FTy.getParamType(1)->isIntegerTy(SizeTBits) && FTy.getParamType(2)->isIntegerTy(32) && - IsSizeTTy(FTy.getParamType(3)) && + FTy.getParamType(3)->isIntegerTy(SizeTBits) && FTy.getParamType(4)->isPointerTy() && FTy.getReturnType()->isIntegerTy(32); @@ -915,16 +1108,17 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_vec_malloc: return (NumParams == 1 && FTy.getReturnType()->isPointerTy()); case LibFunc_memcmp: - return (NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) && - FTy.getParamType(0)->isPointerTy() && - FTy.getParamType(1)->isPointerTy()); + return NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) && + FTy.getParamType(0)->isPointerTy() && + FTy.getParamType(1)->isPointerTy() && + FTy.getParamType(2)->isIntegerTy(SizeTBits); case LibFunc_memchr: case LibFunc_memrchr: return (NumParams == 3 && FTy.getReturnType()->isPointerTy() && FTy.getReturnType() == FTy.getParamType(0) && FTy.getParamType(1)->isIntegerTy(32) && - IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_modf: case LibFunc_modff: case LibFunc_modfl: @@ -934,7 +1128,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_mempcpy_chk: case LibFunc_memmove_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_memcpy: @@ -943,22 +1137,22 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, return (NumParams == 3 && FTy.getReturnType() == FTy.getParamType(0) && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isPointerTy() && - IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_memset_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_memset: return (NumParams == 3 && FTy.getReturnType() == FTy.getParamType(0) && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isIntegerTy() && - IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_memccpy_chk: --NumParams; - if (!IsSizeTTy(FTy.getParamType(NumParams))) + if (!FTy.getParamType(NumParams)->isIntegerTy(SizeTBits)) return false; LLVM_FALLTHROUGH; case LibFunc_memccpy: @@ -970,7 +1164,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_vec_realloc: return (NumParams == 2 && FTy.getReturnType()->isPointerTy() && FTy.getParamType(0) == FTy.getReturnType() && - IsSizeTTy(FTy.getParamType(1))); + FTy.getParamType(1)->isIntegerTy(SizeTBits)); case LibFunc_read: return (NumParams == 3 && FTy.getParamType(1)->isPointerTy()); case LibFunc_rewind: @@ -1051,7 +1245,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, return (NumParams != 0 && FTy.getParamType(0)->isPointerTy()); case LibFunc___kmpc_free_shared: return (NumParams == 2 && FTy.getParamType(0)->isPointerTy() && - IsSizeTTy(FTy.getParamType(1))); + FTy.getParamType(1)->isIntegerTy(SizeTBits)); case LibFunc_fopen: return (NumParams == 2 && FTy.getReturnType()->isPointerTy() && @@ -1141,14 +1335,14 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_vsprintf_chk: return NumParams == 5 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isIntegerTy(32) && - IsSizeTTy(FTy.getParamType(2)) && FTy.getParamType(3)->isPointerTy(); + FTy.getParamType(2)->isIntegerTy(SizeTBits) && FTy.getParamType(3)->isPointerTy(); case LibFunc_vsnprintf: return (NumParams == 4 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(2)->isPointerTy()); case LibFunc_vsnprintf_chk: return NumParams == 6 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(2)->isIntegerTy(32) && - IsSizeTTy(FTy.getParamType(3)) && FTy.getParamType(4)->isPointerTy(); + FTy.getParamType(3)->isIntegerTy(SizeTBits) && FTy.getParamType(4)->isPointerTy(); case LibFunc_open: return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy()); case LibFunc_opendir: @@ -1560,12 +1754,13 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_strnlen: return (NumParams == 2 && FTy.getReturnType() == FTy.getParamType(1) && FTy.getParamType(0)->isPointerTy() && - IsSizeTTy(FTy.getParamType(1))); + FTy.getParamType(1)->isIntegerTy(SizeTBits)); case LibFunc_posix_memalign: return (NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) && FTy.getParamType(0)->isPointerTy() && - IsSizeTTy(FTy.getParamType(1)) && IsSizeTTy(FTy.getParamType(2))); + FTy.getParamType(1)->isIntegerTy(SizeTBits) && + FTy.getParamType(2)->isIntegerTy(SizeTBits)); case LibFunc_wcslen: return (NumParams == 1 && FTy.getParamType(0)->isPointerTy() && @@ -1605,10 +1800,11 @@ bool TargetLibraryInfoImpl::getLibFunc(const Function &FDecl, // avoid string normalization and comparison. if (FDecl.isIntrinsic()) return false; - const DataLayout *DL = - FDecl.getParent() ? &FDecl.getParent()->getDataLayout() : nullptr; + const Module *M = FDecl.getParent(); + assert(M && "Expecting FDecl to be connected to a Module."); + return getLibFunc(FDecl.getName(), F) && - isValidProtoForLibFunc(*FDecl.getFunctionType(), F, DL); + isValidProtoForLibFunc(*FDecl.getFunctionType(), F, *M); } void TargetLibraryInfoImpl::disableAllFunctions() { diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 304d24fe8e4a..5067f493f02d 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -167,11 +167,7 @@ bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, // Note that this block may not be the loop latch block, even if the loop // has a latch block. ExitBlock = BB; - TripCount = SE.getAddExpr(EC, SE.getOne(EC->getType())); - - if (!EC->getType()->isPointerTy() && EC->getType() != CountType) - TripCount = SE.getZeroExtendExpr(TripCount, CountType); - + ExitCount = EC; break; } @@ -263,10 +259,20 @@ bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS, return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS); } +bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace( + unsigned AS) const { + return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS); +} + unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const { return TTIImpl->getAssumedAddrSpace(V); } +std::pair<const Value *, unsigned> +TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const { + return TTIImpl->getPredicatedAddrSpace(V); +} + Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace( IntrinsicInst *II, Value *OldV, Value *NewV) const { return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV); @@ -317,8 +323,9 @@ Optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic( } void TargetTransformInfo::getUnrollingPreferences( - Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const { - return TTIImpl->getUnrollingPreferences(L, SE, UP); + Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP, + OptimizationRemarkEmitter *ORE) const { + return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE); } void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE, @@ -409,6 +416,10 @@ bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const { return TTIImpl->isLegalMaskedExpandLoad(DataType); } +bool TargetTransformInfo::enableOrderedReductions() const { + return TTIImpl->enableOrderedReductions(); +} + bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { return TTIImpl->hasDivRemOp(DataType, IsSigned); } @@ -598,6 +609,10 @@ Optional<unsigned> TargetTransformInfo::getMaxVScale() const { return TTIImpl->getMaxVScale(); } +Optional<unsigned> TargetTransformInfo::getVScaleForTuning() const { + return TTIImpl->getVScaleForTuning(); +} + bool TargetTransformInfo::shouldMaximizeVectorBandwidth() const { return TTIImpl->shouldMaximizeVectorBandwidth(); } @@ -818,6 +833,15 @@ InstructionCost TargetTransformInfo::getVectorInstrCost(unsigned Opcode, return Cost; } +InstructionCost TargetTransformInfo::getReplicationShuffleCost( + Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, + TTI::TargetCostKind CostKind) { + InstructionCost Cost = TTIImpl->getReplicationShuffleCost( + EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind); + assert(Cost >= 0 && "TTI should not produce negative costs!"); + return Cost; +} + InstructionCost TargetTransformInfo::getMemoryOpCost( unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, const Instruction *I) const { diff --git a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp index 20d718f4fad3..23dbb32f38de 100644 --- a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp @@ -521,21 +521,21 @@ static const MDNode *getLeastCommonType(const MDNode *A, const MDNode *B) { return Ret; } -void Instruction::getAAMetadata(AAMDNodes &N, bool Merge) const { - if (Merge) { - N.TBAA = - MDNode::getMostGenericTBAA(N.TBAA, getMetadata(LLVMContext::MD_tbaa)); - N.TBAAStruct = nullptr; - N.Scope = MDNode::getMostGenericAliasScope( - N.Scope, getMetadata(LLVMContext::MD_alias_scope)); - N.NoAlias = - MDNode::intersect(N.NoAlias, getMetadata(LLVMContext::MD_noalias)); - } else { - N.TBAA = getMetadata(LLVMContext::MD_tbaa); - N.TBAAStruct = getMetadata(LLVMContext::MD_tbaa_struct); - N.Scope = getMetadata(LLVMContext::MD_alias_scope); - N.NoAlias = getMetadata(LLVMContext::MD_noalias); - } +AAMDNodes AAMDNodes::merge(const AAMDNodes &Other) const { + AAMDNodes Result; + Result.TBAA = MDNode::getMostGenericTBAA(TBAA, Other.TBAA); + Result.TBAAStruct = nullptr; + Result.Scope = MDNode::getMostGenericAliasScope(Scope, Other.Scope); + Result.NoAlias = MDNode::intersect(NoAlias, Other.NoAlias); + return Result; +} + +AAMDNodes AAMDNodes::concat(const AAMDNodes &Other) const { + AAMDNodes Result; + Result.TBAA = Result.TBAAStruct = nullptr; + Result.Scope = MDNode::getMostGenericAliasScope(Scope, Other.Scope); + Result.NoAlias = MDNode::intersect(NoAlias, Other.NoAlias); + return Result; } static const MDNode *createAccessTag(const MDNode *AccessType) { diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp index f015ba9a09ca..80051fd5f7c1 100644 --- a/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp @@ -126,7 +126,8 @@ void llvm::findDevirtualizableCallsForTypeCheckedLoad( Offset->getZExtValue(), CI, DT); } -Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M) { +Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M, + Constant *TopLevelGlobal) { if (I->getType()->isPointerTy()) { if (Offset == 0) return I; @@ -142,7 +143,8 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M) { unsigned Op = SL->getElementContainingOffset(Offset); return getPointerAtOffset(cast<Constant>(I->getOperand(Op)), - Offset - SL->getElementOffset(Op), M); + Offset - SL->getElementOffset(Op), M, + TopLevelGlobal); } if (auto *C = dyn_cast<ConstantArray>(I)) { ArrayType *VTableTy = C->getType(); @@ -153,7 +155,62 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M) { return nullptr; return getPointerAtOffset(cast<Constant>(I->getOperand(Op)), - Offset % ElemSize, M); + Offset % ElemSize, M, TopLevelGlobal); + } + + // (Swift-specific) relative-pointer support starts here. + if (auto *CI = dyn_cast<ConstantInt>(I)) { + if (Offset == 0 && CI->getZExtValue() == 0) { + return I; + } + } + if (auto *C = dyn_cast<ConstantExpr>(I)) { + switch (C->getOpcode()) { + case Instruction::Trunc: + case Instruction::PtrToInt: + return getPointerAtOffset(cast<Constant>(C->getOperand(0)), Offset, M, + TopLevelGlobal); + case Instruction::Sub: { + auto *Operand0 = cast<Constant>(C->getOperand(0)); + auto *Operand1 = cast<Constant>(C->getOperand(1)); + + auto StripGEP = [](Constant *C) { + auto *CE = dyn_cast<ConstantExpr>(C); + if (!CE) + return C; + if (CE->getOpcode() != Instruction::GetElementPtr) + return C; + return CE->getOperand(0); + }; + auto *Operand1TargetGlobal = StripGEP(getPointerAtOffset(Operand1, 0, M)); + + // Check that in the "sub (@a, @b)" expression, @b points back to the top + // level global (or a GEP thereof) that we're processing. Otherwise bail. + if (Operand1TargetGlobal != TopLevelGlobal) + return nullptr; + + return getPointerAtOffset(Operand0, Offset, M, TopLevelGlobal); + } + default: + return nullptr; + } } return nullptr; } + +void llvm::replaceRelativePointerUsersWithZero(Function *F) { + for (auto *U : F->users()) { + auto *PtrExpr = dyn_cast<ConstantExpr>(U); + if (!PtrExpr || PtrExpr->getOpcode() != Instruction::PtrToInt) + continue; + + for (auto *PtrToIntUser : PtrExpr->users()) { + auto *SubExpr = dyn_cast<ConstantExpr>(PtrToIntUser); + if (!SubExpr || SubExpr->getOpcode() != Instruction::Sub) + continue; + + SubExpr->replaceNonMetadataUsesWith( + ConstantInt::get(SubExpr->getType(), 0)); + } + } +} diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 522d21812c6a..1c41c77a8cfb 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -84,6 +84,17 @@ using namespace llvm::PatternMatch; static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses", cl::Hidden, cl::init(20)); +// According to the LangRef, branching on a poison condition is absolutely +// immediate full UB. However, historically we haven't implemented that +// consistently as we have an important transformation (non-trivial unswitch) +// which introduces instances of branch on poison/undef to otherwise well +// defined programs. This flag exists to let us test optimization benefit +// of exploiting the specified behavior (in combination with enabling the +// unswitch fix.) +static cl::opt<bool> BranchOnPoisonAsUB("branch-on-poison-as-ub", + cl::Hidden, cl::init(false)); + + /// Returns the bitwidth of the given scalar or pointer type. For vector types, /// returns the element type's bitwidth. static unsigned getBitWidth(Type *Ty, const DataLayout &DL) { @@ -165,8 +176,8 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf, int NumElts = cast<FixedVectorType>(Shuf->getOperand(0)->getType())->getNumElements(); int NumMaskElts = cast<FixedVectorType>(Shuf->getType())->getNumElements(); - DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts); - if (DemandedElts.isNullValue()) + DemandedLHS = DemandedRHS = APInt::getZero(NumElts); + if (DemandedElts.isZero()) return true; // Simple case of a shuffle with zeroinitializer. if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) { @@ -206,7 +217,7 @@ static void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, auto *FVTy = dyn_cast<FixedVectorType>(V->getType()); APInt DemandedElts = - FVTy ? APInt::getAllOnesValue(FVTy->getNumElements()) : APInt(1, 1); + FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1); computeKnownBits(V, DemandedElts, Known, Depth, Q); } @@ -279,16 +290,11 @@ bool llvm::haveNoCommonBitsSet(const Value *LHS, const Value *RHS, return KnownBits::haveNoCommonBitsSet(LHSKnown, RHSKnown); } -bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI) { - for (const User *U : CxtI->users()) { - if (const ICmpInst *IC = dyn_cast<ICmpInst>(U)) - if (IC->isEquality()) - if (Constant *C = dyn_cast<Constant>(IC->getOperand(1))) - if (C->isNullValue()) - continue; - return false; - } - return true; +bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) { + return !I->user_empty() && all_of(I->users(), [](const User *U) { + ICmpInst::Predicate P; + return match(U, m_ICmp(P, m_Value(), m_Zero())) && ICmpInst::isEquality(P); + }); } static bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth, @@ -378,7 +384,7 @@ static unsigned ComputeNumSignBits(const Value *V, unsigned Depth, auto *FVTy = dyn_cast<FixedVectorType>(V->getType()); APInt DemandedElts = - FVTy ? APInt::getAllOnesValue(FVTy->getNumElements()) : APInt(1, 1); + FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1); return ComputeNumSignBits(V, DemandedElts, Depth, Q); } @@ -390,6 +396,14 @@ unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL, V, Depth, Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo)); } +unsigned llvm::ComputeMinSignedBits(const Value *V, const DataLayout &DL, + unsigned Depth, AssumptionCache *AC, + const Instruction *CxtI, + const DominatorTree *DT) { + unsigned SignBits = ComputeNumSignBits(V, DL, Depth, AC, CxtI, DT); + return V->getType()->getScalarSizeInBits() - SignBits + 1; +} + static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1, bool NSW, const APInt &DemandedElts, KnownBits &KnownOut, KnownBits &Known2, @@ -499,7 +513,9 @@ static bool isEphemeralValueOf(const Instruction *I, const Value *E) { if (V == E) return true; - if (V == I || isSafeToSpeculativelyExecute(V)) { + if (V == I || (isa<Instruction>(V) && + !cast<Instruction>(V)->mayHaveSideEffects() && + !cast<Instruction>(V)->isTerminator())) { EphValues.insert(V); if (const User *U = dyn_cast<User>(V)) append_range(WorkSet, U->operands()); @@ -547,10 +563,9 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, // We limit the scan distance between the assume and its context instruction // to avoid a compile-time explosion. This limit is chosen arbitrarily, so // it can be adjusted if needed (could be turned into a cl::opt). - unsigned ScanLimit = 15; - for (BasicBlock::const_iterator I(CxtI), IE(Inv); I != IE; ++I) - if (!isGuaranteedToTransferExecutionToSuccessor(&*I) || --ScanLimit == 0) - return false; + auto Range = make_range(CxtI->getIterator(), Inv->getIterator()); + if (!isGuaranteedToTransferExecutionToSuccessor(Range, 15)) + return false; return !isEphemeralValueOf(Inv, CxtI); } @@ -582,7 +597,7 @@ static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) { return false; ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, *C); - return !TrueValues.contains(APInt::getNullValue(C->getBitWidth())); + return !TrueValues.contains(APInt::getZero(C->getBitWidth())); } static bool isKnownNonZeroFromAssume(const Value *V, const Query &Q) { @@ -641,7 +656,7 @@ static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known, if (V->getType()->isPointerTy()) { if (RetainedKnowledge RK = getKnowledgeValidInContext( V, {Attribute::Alignment}, Q.CxtI, Q.DT, Q.AC)) { - Known.Zero.setLowBits(Log2_32(RK.ArgValue)); + Known.Zero.setLowBits(Log2_64(RK.ArgValue)); } } @@ -1210,7 +1225,7 @@ static void computeKnownBitsFromOperator(const Operator *I, // (dependent on endian) to form the full result of known bits. unsigned NumElts = DemandedElts.getBitWidth(); unsigned SubScale = BitWidth / SubBitWidth; - APInt SubDemandedElts = APInt::getNullValue(NumElts * SubScale); + APInt SubDemandedElts = APInt::getZero(NumElts * SubScale); for (unsigned i = 0; i != NumElts; ++i) { if (DemandedElts[i]) SubDemandedElts.setBit(i * SubScale); @@ -1383,7 +1398,7 @@ static void computeKnownBitsFromOperator(const Operator *I, Known = KnownBits::computeForAddSub( /*Add=*/true, /*NSW=*/false, Known, IndexBits); } - if (!Known.isUnknown() && !AccConstIndices.isNullValue()) { + if (!Known.isUnknown() && !AccConstIndices.isZero()) { KnownBits Index = KnownBits::makeConstant(AccConstIndices); Known = KnownBits::computeForAddSub( /*Add=*/true, /*NSW=*/false, Known, Index); @@ -1512,7 +1527,7 @@ static void computeKnownBitsFromOperator(const Operator *I, // taking conservative care to avoid excessive recursion. if (Depth < MaxAnalysisRecursionDepth - 1 && !Known.Zero && !Known.One) { // Skip if every incoming value references to ourself. - if (dyn_cast_or_null<UndefValue>(P->hasConstantValue())) + if (isa_and_nonnull<UndefValue>(P->hasConstantValue())) break; Known.Zero.setAllBits(); @@ -1689,6 +1704,33 @@ static void computeKnownBitsFromOperator(const Operator *I, if (BitWidth >= 32) Known.Zero.setBitsFrom(31); break; + case Intrinsic::vscale: { + if (!II->getParent() || !II->getFunction() || + !II->getFunction()->hasFnAttribute(Attribute::VScaleRange)) + break; + + auto VScaleRange = II->getFunction() + ->getFnAttribute(Attribute::VScaleRange) + .getVScaleRangeArgs(); + + if (VScaleRange.second == 0) + break; + + // If vscale min = max then we know the exact value at compile time + // and hence we know the exact bits. + if (VScaleRange.first == VScaleRange.second) { + Known.One = VScaleRange.first; + Known.Zero = VScaleRange.first; + Known.Zero.flipAllBits(); + break; + } + + unsigned FirstZeroHighBit = 32 - countLeadingZeros(VScaleRange.second); + if (FirstZeroHighBit < BitWidth) + Known.Zero.setBitsFrom(FirstZeroHighBit); + + break; + } } } break; @@ -1763,7 +1805,7 @@ static void computeKnownBitsFromOperator(const Operator *I, break; } unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements(); - APInt DemandedVecElts = APInt::getAllOnesValue(NumElts); + APInt DemandedVecElts = APInt::getAllOnes(NumElts); if (CIdx && CIdx->getValue().ult(NumElts)) DemandedVecElts = APInt::getOneBitSet(NumElts, CIdx->getZExtValue()); computeKnownBits(Vec, DemandedVecElts, Known, Depth + 1, Q); @@ -2248,7 +2290,7 @@ static bool isNonZeroRecurrence(const PHINode *PN) { Value *Start = nullptr, *Step = nullptr; const APInt *StartC, *StepC; if (!matchSimpleRecurrence(PN, BO, Start, Step) || - !match(Start, m_APInt(StartC)) || StartC->isNullValue()) + !match(Start, m_APInt(StartC)) || StartC->isZero()) return false; switch (BO->getOpcode()) { @@ -2260,7 +2302,7 @@ static bool isNonZeroRecurrence(const PHINode *PN) { StartC->isNegative() == StepC->isNegative()); case Instruction::Mul: return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) && - match(Step, m_APInt(StepC)) && !StepC->isNullValue(); + match(Step, m_APInt(StepC)) && !StepC->isZero(); case Instruction::Shl: return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap(); case Instruction::AShr: @@ -2532,7 +2574,7 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth, auto *CIdx = dyn_cast<ConstantInt>(Idx); if (auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType())) { unsigned NumElts = VecTy->getNumElements(); - APInt DemandedVecElts = APInt::getAllOnesValue(NumElts); + APInt DemandedVecElts = APInt::getAllOnes(NumElts); if (CIdx && CIdx->getValue().ult(NumElts)) DemandedVecElts = APInt::getOneBitSet(NumElts, CIdx->getZExtValue()); return isKnownNonZero(Vec, DemandedVecElts, Depth, Q); @@ -2559,7 +2601,7 @@ bool isKnownNonZero(const Value* V, unsigned Depth, const Query& Q) { auto *FVTy = dyn_cast<FixedVectorType>(V->getType()); APInt DemandedElts = - FVTy ? APInt::getAllOnesValue(FVTy->getNumElements()) : APInt(1, 1); + FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1); return isKnownNonZero(V, DemandedElts, Depth, Q); } @@ -2694,8 +2736,7 @@ static bool isNonEqualMul(const Value *V1, const Value *V2, unsigned Depth, const APInt *C; return match(OBO, m_Mul(m_Specific(V1), m_APInt(C))) && (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) && - !C->isNullValue() && !C->isOneValue() && - isKnownNonZero(V1, Depth + 1, Q); + !C->isZero() && !C->isOne() && isKnownNonZero(V1, Depth + 1, Q); } return false; } @@ -2708,7 +2749,7 @@ static bool isNonEqualShl(const Value *V1, const Value *V2, unsigned Depth, const APInt *C; return match(OBO, m_Shl(m_Specific(V1), m_APInt(C))) && (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) && - !C->isNullValue() && isKnownNonZero(V1, Depth + 1, Q); + !C->isZero() && isKnownNonZero(V1, Depth + 1, Q); } return false; } @@ -3051,7 +3092,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V, // If the input is known to be 0 or 1, the output is 0/-1, which is // all sign bits set. - if ((Known.Zero | 1).isAllOnesValue()) + if ((Known.Zero | 1).isAllOnes()) return TyBits; // If we are subtracting one from a positive number, there is no carry @@ -3075,7 +3116,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V, computeKnownBits(U->getOperand(1), Known, Depth + 1, Q); // If the input is known to be 0 or 1, the output is 0/-1, which is // all sign bits set. - if ((Known.Zero | 1).isAllOnesValue()) + if ((Known.Zero | 1).isAllOnes()) return TyBits; // If the input is known to be positive (the sign bit is known clear), @@ -4533,6 +4574,12 @@ AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) { if (OffsetZero && !GEP->hasAllZeroIndices()) return nullptr; AddWork(GEP->getPointerOperand()); + } else if (CallBase *CB = dyn_cast<CallBase>(V)) { + Value *Returned = CB->getReturnedArgOperand(); + if (Returned) + AddWork(Returned); + else + return nullptr; } else { return nullptr; } @@ -4614,7 +4661,7 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V, if (*Denominator == 0) return false; // It's safe to hoist if the denominator is not 0 or -1. - if (!Denominator->isAllOnesValue()) + if (!Denominator->isAllOnes()) return true; // At this point we know that the denominator is -1. It is safe to hoist as // long we know that the numerator is not INT_MIN. @@ -4922,15 +4969,14 @@ bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO, return llvm::any_of(GuardingBranches, AllUsesGuardedByBranch); } -static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly) { - // See whether I has flags that may create poison - if (const auto *OvOp = dyn_cast<OverflowingBinaryOperator>(Op)) { - if (OvOp->hasNoSignedWrap() || OvOp->hasNoUnsignedWrap()) - return true; - } - if (const auto *ExactOp = dyn_cast<PossiblyExactOperator>(Op)) - if (ExactOp->isExact()) - return true; +static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly, + bool ConsiderFlags) { + + if (ConsiderFlags && Op->hasPoisonGeneratingFlags()) + return true; + + // TODO: this should really be under the ConsiderFlags block, but currently + // these are not dropped by dropPoisonGeneratingFlags if (const auto *FP = dyn_cast<FPMathOperator>(Op)) { auto FMF = FP->getFastMathFlags(); if (FMF.noNaNs() || FMF.noInfs()) @@ -5019,10 +5065,10 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly) { case Instruction::ICmp: case Instruction::FCmp: return false; - case Instruction::GetElementPtr: { - const auto *GEP = cast<GEPOperator>(Op); - return GEP->isInBounds(); - } + case Instruction::GetElementPtr: + // inbounds is handled above + // TODO: what about inrange on constexpr? + return false; default: { const auto *CE = dyn_cast<ConstantExpr>(Op); if (isa<CastInst>(Op) || (CE && CE->isCast())) @@ -5035,12 +5081,12 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly) { } } -bool llvm::canCreateUndefOrPoison(const Operator *Op) { - return ::canCreateUndefOrPoison(Op, /*PoisonOnly=*/false); +bool llvm::canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlags) { + return ::canCreateUndefOrPoison(Op, /*PoisonOnly=*/false, ConsiderFlags); } -bool llvm::canCreatePoison(const Operator *Op) { - return ::canCreateUndefOrPoison(Op, /*PoisonOnly=*/true); +bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlags) { + return ::canCreateUndefOrPoison(Op, /*PoisonOnly=*/true, ConsiderFlags); } static bool directlyImpliesPoison(const Value *ValAssumedPoison, @@ -5068,7 +5114,7 @@ static bool directlyImpliesPoison(const Value *ValAssumedPoison, const WithOverflowInst *II; if (match(I, m_ExtractValue(m_WithOverflowInst(II))) && (match(ValAssumedPoison, m_ExtractValue(m_Specific(II))) || - llvm::is_contained(II->arg_operands(), ValAssumedPoison))) + llvm::is_contained(II->args(), ValAssumedPoison))) return true; } return false; @@ -5225,8 +5271,7 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, Dominator = Dominator->getIDom(); } - SmallVector<Attribute::AttrKind, 2> AttrKinds{Attribute::NoUndef}; - if (getKnowledgeValidInContext(V, AttrKinds, CtxI, DT, AC)) + if (getKnowledgeValidInContext(V, {Attribute::NoUndef}, CtxI, DT, AC)) return true; return false; @@ -5304,6 +5349,27 @@ bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) { return true; } +bool llvm::isGuaranteedToTransferExecutionToSuccessor( + BasicBlock::const_iterator Begin, BasicBlock::const_iterator End, + unsigned ScanLimit) { + return isGuaranteedToTransferExecutionToSuccessor(make_range(Begin, End), + ScanLimit); +} + +bool llvm::isGuaranteedToTransferExecutionToSuccessor( + iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) { + assert(ScanLimit && "scan limit must be non-zero"); + for (const Instruction &I : Range) { + if (isa<DbgInfoIntrinsic>(I)) + continue; + if (--ScanLimit == 0) + return false; + if (!isGuaranteedToTransferExecutionToSuccessor(&I)) + return false; + } + return true; +} + bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I, const Loop *L) { // The loop header is guaranteed to be executed for every iteration. @@ -5391,7 +5457,10 @@ void llvm::getGuaranteedWellDefinedOps( } break; } - + case Instruction::Ret: + if (I->getFunction()->hasRetAttribute(Attribute::NoUndef)) + Operands.insert(I->getOperand(0)); + break; default: break; } @@ -5408,7 +5477,16 @@ void llvm::getGuaranteedNonPoisonOps(const Instruction *I, case Instruction::SRem: Operands.insert(I->getOperand(1)); break; - + case Instruction::Switch: + if (BranchOnPoisonAsUB) + Operands.insert(cast<SwitchInst>(I)->getCondition()); + break; + case Instruction::Br: { + auto *BR = cast<BranchInst>(I); + if (BranchOnPoisonAsUB && BR->isConditional()) + Operands.insert(BR->getCondition()); + break; + } default: break; } @@ -5835,15 +5913,13 @@ static SelectPatternResult matchMinMax(CmpInst::Predicate Pred, // Is the sign bit set? // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN - if (Pred == CmpInst::ICMP_SLT && C1->isNullValue() && - C2->isMaxSignedValue()) + if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue()) return {CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false}; // Is the sign bit clear? // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN - if (Pred == CmpInst::ICMP_SGT && C1->isAllOnesValue() && - C2->isMinSignedValue()) + if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue()) return {CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false}; } @@ -6253,6 +6329,16 @@ CmpInst::Predicate llvm::getInverseMinMaxPred(SelectPatternFlavor SPF) { return getMinMaxPred(getInverseMinMaxFlavor(SPF)); } +APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) { + switch (SPF) { + case SPF_SMAX: return APInt::getSignedMaxValue(BitWidth); + case SPF_SMIN: return APInt::getSignedMinValue(BitWidth); + case SPF_UMAX: return APInt::getMaxValue(BitWidth); + case SPF_UMIN: return APInt::getMinValue(BitWidth); + default: llvm_unreachable("Unexpected flavor"); + } +} + std::pair<Intrinsic::ID, bool> llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) { // Check if VL contains select instructions that can be folded into a min/max @@ -6681,7 +6767,7 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, const APInt *C; switch (BO.getOpcode()) { case Instruction::Add: - if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { + if (match(BO.getOperand(1), m_APInt(C)) && !C->isZero()) { // FIXME: If we have both nuw and nsw, we should reduce the range further. if (IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(&BO))) { // 'add nuw x, C' produces [C, UINT_MAX]. @@ -6719,7 +6805,7 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, Upper = APInt::getSignedMaxValue(Width).ashr(*C) + 1; } else if (match(BO.getOperand(0), m_APInt(C))) { unsigned ShiftAmount = Width - 1; - if (!C->isNullValue() && IIQ.isExact(&BO)) + if (!C->isZero() && IIQ.isExact(&BO)) ShiftAmount = C->countTrailingZeros(); if (C->isNegative()) { // 'ashr C, x' produces [C, C >> (Width-1)] @@ -6736,11 +6822,11 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, case Instruction::LShr: if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { // 'lshr x, C' produces [0, UINT_MAX >> C]. - Upper = APInt::getAllOnesValue(Width).lshr(*C) + 1; + Upper = APInt::getAllOnes(Width).lshr(*C) + 1; } else if (match(BO.getOperand(0), m_APInt(C))) { // 'lshr C, x' produces [C >> (Width-1), C]. unsigned ShiftAmount = Width - 1; - if (!C->isNullValue() && IIQ.isExact(&BO)) + if (!C->isZero() && IIQ.isExact(&BO)) ShiftAmount = C->countTrailingZeros(); Lower = C->lshr(ShiftAmount); Upper = *C + 1; @@ -6773,7 +6859,7 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, if (match(BO.getOperand(1), m_APInt(C))) { APInt IntMin = APInt::getSignedMinValue(Width); APInt IntMax = APInt::getSignedMaxValue(Width); - if (C->isAllOnesValue()) { + if (C->isAllOnes()) { // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX] // where C != -1 and C != 0 and C != 1 Lower = IntMin + 1; @@ -6802,7 +6888,7 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, break; case Instruction::UDiv: - if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { + if (match(BO.getOperand(1), m_APInt(C)) && !C->isZero()) { // 'udiv x, C' produces [0, UINT_MAX / C]. Upper = APInt::getMaxValue(Width).udiv(*C) + 1; } else if (match(BO.getOperand(0), m_APInt(C))) { @@ -6946,7 +7032,7 @@ static void setLimitsForSelectPattern(const SelectInst &SI, APInt &Lower, // If the negation part of the abs (in RHS) has the NSW flag, // then the result of abs(X) is [0..SIGNED_MAX], // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN. - Lower = APInt::getNullValue(BitWidth); + Lower = APInt::getZero(BitWidth); if (match(RHS, m_Neg(m_Specific(LHS))) && IIQ.hasNoSignedWrap(cast<Instruction>(RHS))) Upper = APInt::getSignedMaxValue(BitWidth) + 1; @@ -6986,9 +7072,27 @@ static void setLimitsForSelectPattern(const SelectInst &SI, APInt &Lower, } } +static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) { + // The maximum representable value of a half is 65504. For floats the maximum + // value is 3.4e38 which requires roughly 129 bits. + unsigned BitWidth = I->getType()->getScalarSizeInBits(); + if (!I->getOperand(0)->getType()->getScalarType()->isHalfTy()) + return; + if (isa<FPToSIInst>(I) && BitWidth >= 17) { + Lower = APInt(BitWidth, -65504); + Upper = APInt(BitWidth, 65505); + } + + if (isa<FPToUIInst>(I) && BitWidth >= 16) { + // For a fptoui the lower limit is left as 0. + Upper = APInt(BitWidth, 65505); + } +} + ConstantRange llvm::computeConstantRange(const Value *V, bool UseInstrInfo, AssumptionCache *AC, const Instruction *CtxI, + const DominatorTree *DT, unsigned Depth) { assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction"); @@ -7009,6 +7113,8 @@ ConstantRange llvm::computeConstantRange(const Value *V, bool UseInstrInfo, setLimitsForIntrinsic(*II, Lower, Upper); else if (auto *SI = dyn_cast<SelectInst>(V)) setLimitsForSelectPattern(*SI, Lower, Upper, IIQ); + else if (isa<FPToUIInst>(V) || isa<FPToSIInst>(V)) + setLimitForFPToI(cast<Instruction>(V), Lower, Upper); ConstantRange CR = ConstantRange::getNonEmpty(Lower, Upper); @@ -7027,7 +7133,7 @@ ConstantRange llvm::computeConstantRange(const Value *V, bool UseInstrInfo, assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && "must be an assume intrinsic"); - if (!isValidAssumeForContext(I, CtxI, nullptr)) + if (!isValidAssumeForContext(I, CtxI, DT)) continue; Value *Arg = I->getArgOperand(0); ICmpInst *Cmp = dyn_cast<ICmpInst>(Arg); @@ -7035,9 +7141,9 @@ ConstantRange llvm::computeConstantRange(const Value *V, bool UseInstrInfo, if (!Cmp || Cmp->getOperand(0) != V) continue; ConstantRange RHS = computeConstantRange(Cmp->getOperand(1), UseInstrInfo, - AC, I, Depth + 1); + AC, I, DT, Depth + 1); CR = CR.intersectWith( - ConstantRange::makeSatisfyingICmpRegion(Cmp->getPredicate(), RHS)); + ConstantRange::makeAllowedICmpRegion(Cmp->getPredicate(), RHS)); } } diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 0a14a1432934..655c248907f6 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -331,6 +331,12 @@ Value *llvm::findScalarElement(Value *V, unsigned EltNo) { if (Elt->isNullValue()) return findScalarElement(Val, EltNo); + // If the vector is a splat then we can trivially find the scalar element. + if (isa<ScalableVectorType>(VTy)) + if (Value *Splat = getSplatValue(V)) + if (EltNo < VTy->getElementCount().getKnownMinValue()) + return Splat; + // Otherwise, we don't know. return nullptr; } @@ -824,6 +830,23 @@ llvm::SmallVector<int, 16> llvm::createSequentialMask(unsigned Start, return Mask; } +llvm::SmallVector<int, 16> llvm::createUnaryMask(ArrayRef<int> Mask, + unsigned NumElts) { + // Avoid casts in the loop and make sure we have a reasonable number. + int NumEltsSigned = NumElts; + assert(NumEltsSigned > 0 && "Expected smaller or non-zero element count"); + + // If the mask chooses an element from operand 1, reduce it to choose from the + // corresponding element of operand 0. Undef mask elements are unchanged. + SmallVector<int, 16> UnaryMask; + for (int MaskElt : Mask) { + assert((MaskElt < NumEltsSigned * 2) && "Expected valid shuffle mask"); + int UnaryElt = MaskElt >= NumEltsSigned ? MaskElt - NumEltsSigned : MaskElt; + UnaryMask.push_back(UnaryElt); + } + return UnaryMask; +} + /// A helper function for concatenating vectors. This function concatenates two /// vectors having the same element type. If the second vector has fewer /// elements than the first, it is padded with undefs. @@ -940,7 +963,7 @@ APInt llvm::possiblyDemandedEltsInMask(Value *Mask) { const unsigned VWidth = cast<FixedVectorType>(Mask->getType())->getNumElements(); - APInt DemandedElts = APInt::getAllOnesValue(VWidth); + APInt DemandedElts = APInt::getAllOnes(VWidth); if (auto *CV = dyn_cast<ConstantVector>(Mask)) for (unsigned i = 0; i < VWidth; i++) if (CV->getAggregateElement(i)->isNullValue()) @@ -980,7 +1003,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses( // wrap around the address space we would do a memory access at nullptr // even without the transformation. The wrapping checks are therefore // deferred until after we've formed the interleaved groups. - int64_t Stride = getPtrStride(PSE, Ptr, TheLoop, Strides, + int64_t Stride = getPtrStride(PSE, ElementTy, Ptr, TheLoop, Strides, /*Assume=*/true, /*ShouldCheckWrap=*/false); const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr); @@ -1193,15 +1216,24 @@ void InterleavedAccessInfo::analyzeInterleaving( } // Iteration over A accesses. } // Iteration over B accesses. - // Remove interleaved store groups with gaps. - for (auto *Group : StoreGroups) - if (Group->getNumMembers() != Group->getFactor()) { - LLVM_DEBUG( - dbgs() << "LV: Invalidate candidate interleaved store group due " - "to gaps.\n"); - releaseGroup(Group); - } - // Remove interleaved groups with gaps (currently only loads) whose memory + auto InvalidateGroupIfMemberMayWrap = [&](InterleaveGroup<Instruction> *Group, + int Index, + std::string FirstOrLast) -> bool { + Instruction *Member = Group->getMember(Index); + assert(Member && "Group member does not exist"); + Value *MemberPtr = getLoadStorePointerOperand(Member); + Type *AccessTy = getLoadStoreType(Member); + if (getPtrStride(PSE, AccessTy, MemberPtr, TheLoop, Strides, + /*Assume=*/false, /*ShouldCheckWrap=*/true)) + return false; + LLVM_DEBUG(dbgs() << "LV: Invalidate candidate interleaved group due to " + << FirstOrLast + << " group member potentially pointer-wrapping.\n"); + releaseGroup(Group); + return true; + }; + + // Remove interleaved groups with gaps whose memory // accesses may wrap around. We have to revisit the getPtrStride analysis, // this time with ShouldCheckWrap=true, since collectConstStrideAccesses does // not check wrapping (see documentation there). @@ -1227,26 +1259,12 @@ void InterleavedAccessInfo::analyzeInterleaving( // So we check only group member 0 (which is always guaranteed to exist), // and group member Factor - 1; If the latter doesn't exist we rely on // peeling (if it is a non-reversed accsess -- see Case 3). - Value *FirstMemberPtr = getLoadStorePointerOperand(Group->getMember(0)); - if (!getPtrStride(PSE, FirstMemberPtr, TheLoop, Strides, /*Assume=*/false, - /*ShouldCheckWrap=*/true)) { - LLVM_DEBUG( - dbgs() << "LV: Invalidate candidate interleaved group due to " - "first group member potentially pointer-wrapping.\n"); - releaseGroup(Group); + if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string("first"))) continue; - } - Instruction *LastMember = Group->getMember(Group->getFactor() - 1); - if (LastMember) { - Value *LastMemberPtr = getLoadStorePointerOperand(LastMember); - if (!getPtrStride(PSE, LastMemberPtr, TheLoop, Strides, /*Assume=*/false, - /*ShouldCheckWrap=*/true)) { - LLVM_DEBUG( - dbgs() << "LV: Invalidate candidate interleaved group due to " - "last group member potentially pointer-wrapping.\n"); - releaseGroup(Group); - } - } else { + if (Group->getMember(Group->getFactor() - 1)) + InvalidateGroupIfMemberMayWrap(Group, Group->getFactor() - 1, + std::string("last")); + else { // Case 3: A non-reversed interleaved load group with gaps: We need // to execute at least one scalar epilogue iteration. This will ensure // we don't speculatively access memory out-of-bounds. We only need @@ -1264,6 +1282,39 @@ void InterleavedAccessInfo::analyzeInterleaving( RequiresScalarEpilogue = true; } } + + for (auto *Group : StoreGroups) { + // Case 1: A full group. Can Skip the checks; For full groups, if the wide + // store would wrap around the address space we would do a memory access at + // nullptr even without the transformation. + if (Group->getNumMembers() == Group->getFactor()) + continue; + + // Interleave-store-group with gaps is implemented using masked wide store. + // Remove interleaved store groups with gaps if + // masked-interleaved-accesses are not enabled by the target. + if (!EnablePredicatedInterleavedMemAccesses) { + LLVM_DEBUG( + dbgs() << "LV: Invalidate candidate interleaved store group due " + "to gaps.\n"); + releaseGroup(Group); + continue; + } + + // Case 2: If first and last members of the group don't wrap this implies + // that all the pointers in the group don't wrap. + // So we check only group member 0 (which is always guaranteed to exist), + // and the last group member. Case 3 (scalar epilog) is not relevant for + // stores with gaps, which are implemented with masked-store (rather than + // speculative access, as in loads). + if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string("first"))) + continue; + for (int Index = Group->getFactor() - 1; Index > 0; Index--) + if (Group->getMember(Index)) { + InvalidateGroupIfMemberMayWrap(Group, Index, std::string("last")); + break; + } + } } void InterleavedAccessInfo::invalidateGroupsRequiringScalarEpilogue() { @@ -1325,9 +1376,7 @@ std::string VFABI::mangleTLIVectorName(StringRef VectorName, void VFABI::getVectorVariantNames( const CallInst &CI, SmallVectorImpl<std::string> &VariantMappings) { - const StringRef S = - CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName) - .getValueAsString(); + const StringRef S = CI.getFnAttr(VFABI::MappingsAttrName).getValueAsString(); if (S.empty()) return; |