diff options
Diffstat (limited to 'llvm/lib/Transforms/IPO/AttributorAttributes.cpp')
| -rw-r--r-- | llvm/lib/Transforms/IPO/AttributorAttributes.cpp | 2006 |
1 files changed, 1144 insertions, 862 deletions
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 2d88e329e093..4d99ce7e3175 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -14,9 +14,11 @@ #include "llvm/Transforms/IPO/Attributor.h" #include "llvm/ADT/APInt.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -30,21 +32,29 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/Argument.h" #include "llvm/IR/Assumptions.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/NoFolder.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" +#include "llvm/Support/GraphWriter.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ValueMapper.h" #include <cassert> using namespace llvm; @@ -69,11 +79,11 @@ static cl::opt<unsigned, true> MaxPotentialValues( cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues), cl::init(7)); -static cl::opt<unsigned> - MaxInterferingWrites("attributor-max-interfering-writes", cl::Hidden, - cl::desc("Maximum number of interfering writes to " - "check before assuming all might interfere."), - cl::init(6)); +static cl::opt<unsigned> MaxInterferingAccesses( + "attributor-max-interfering-accesses", cl::Hidden, + cl::desc("Maximum number of interfering accesses to " + "check before assuming all might interfere."), + cl::init(6)); STATISTIC(NumAAs, "Number of abstract attributes created"); @@ -140,6 +150,7 @@ PIPE_OPERATOR(AANonNull) PIPE_OPERATOR(AANoAlias) PIPE_OPERATOR(AADereferenceable) PIPE_OPERATOR(AAAlign) +PIPE_OPERATOR(AAInstanceInfo) PIPE_OPERATOR(AANoCapture) PIPE_OPERATOR(AAValueSimplify) PIPE_OPERATOR(AANoFree) @@ -150,7 +161,7 @@ PIPE_OPERATOR(AAMemoryLocation) PIPE_OPERATOR(AAValueConstantRange) PIPE_OPERATOR(AAPrivatizablePtr) PIPE_OPERATOR(AAUndefinedBehavior) -PIPE_OPERATOR(AAPotentialValues) +PIPE_OPERATOR(AAPotentialConstantValues) PIPE_OPERATOR(AANoUndef) PIPE_OPERATOR(AACallEdges) PIPE_OPERATOR(AAFunctionReachability) @@ -170,6 +181,45 @@ ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S, } // namespace llvm +/// Checks if a type could have padding bytes. +static bool isDenselyPacked(Type *Ty, const DataLayout &DL) { + // There is no size information, so be conservative. + if (!Ty->isSized()) + return false; + + // If the alloc size is not equal to the storage size, then there are padding + // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128. + if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty)) + return false; + + // FIXME: This isn't the right way to check for padding in vectors with + // non-byte-size elements. + if (VectorType *SeqTy = dyn_cast<VectorType>(Ty)) + return isDenselyPacked(SeqTy->getElementType(), DL); + + // For array types, check for padding within members. + if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty)) + return isDenselyPacked(SeqTy->getElementType(), DL); + + if (!isa<StructType>(Ty)) + return true; + + // Check for padding within and between elements of a struct. + StructType *StructTy = cast<StructType>(Ty); + const StructLayout *Layout = DL.getStructLayout(StructTy); + uint64_t StartPos = 0; + for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) { + Type *ElTy = StructTy->getElementType(I); + if (!isDenselyPacked(ElTy, DL)) + return false; + if (StartPos != Layout->getElementOffsetInBits(I)) + return false; + StartPos += DL.getTypeAllocSizeInBits(ElTy); + } + + return true; +} + /// Get pointer operand of memory accessing instruction. If \p I is /// not a memory accessing instruction, return nullptr. If \p AllowVolatile, /// is set to false and the instruction is volatile, return nullptr. @@ -236,7 +286,8 @@ static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, } // Ensure the result has the requested type. - Ptr = IRB.CreateBitOrPointerCast(Ptr, ResTy, Ptr->getName() + ".cast"); + Ptr = IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, ResTy, + Ptr->getName() + ".cast"); LLVM_DEBUG(dbgs() << "Constructed pointer: " << *Ptr << "\n"); return Ptr; @@ -251,25 +302,32 @@ static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, /// once. Note that the value used for the callback may still be the value /// associated with \p IRP (due to PHIs). To limit how much effort is invested, /// we will never visit more values than specified by \p MaxValues. -/// If \p Intraprocedural is set to true only values valid in the scope of -/// \p CtxI will be visited and simplification into other scopes is prevented. +/// If \p VS does not contain the Interprocedural bit, only values valid in the +/// scope of \p CtxI will be visited and simplification into other scopes is +/// prevented. template <typename StateTy> static bool genericValueTraversal( Attributor &A, IRPosition IRP, const AbstractAttribute &QueryingAA, StateTy &State, function_ref<bool(Value &, const Instruction *, StateTy &, bool)> VisitValueCB, - const Instruction *CtxI, bool UseValueSimplify = true, int MaxValues = 16, + const Instruction *CtxI, bool &UsedAssumedInformation, + bool UseValueSimplify = true, int MaxValues = 16, function_ref<Value *(Value *)> StripCB = nullptr, - bool Intraprocedural = false) { + AA::ValueScope VS = AA::Interprocedural) { - const AAIsDead *LivenessAA = nullptr; - if (IRP.getAnchorScope()) - LivenessAA = &A.getAAFor<AAIsDead>( - QueryingAA, - IRPosition::function(*IRP.getAnchorScope(), IRP.getCallBaseContext()), - DepClassTy::NONE); - bool AnyDead = false; + struct LivenessInfo { + const AAIsDead *LivenessAA = nullptr; + bool AnyDead = false; + }; + SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs; + auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & { + LivenessInfo &LI = LivenessAAs[&F]; + if (!LI.LivenessAA) + LI.LivenessAA = &A.getAAFor<AAIsDead>(QueryingAA, IRPosition::function(F), + DepClassTy::NONE); + return LI; + }; Value *InitialV = &IRP.getAssociatedValue(); using Item = std::pair<Value *, const Instruction *>; @@ -319,10 +377,9 @@ static bool genericValueTraversal( // Look through select instructions, visit assumed potential values. if (auto *SI = dyn_cast<SelectInst>(V)) { - bool UsedAssumedInformation = false; Optional<Constant *> C = A.getAssumedConstant( *SI->getCondition(), QueryingAA, UsedAssumedInformation); - bool NoValueYet = !C.hasValue(); + bool NoValueYet = !C; if (NoValueYet || isa_and_nonnull<UndefValue>(*C)) continue; if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) { @@ -340,12 +397,12 @@ static bool genericValueTraversal( // Look through phi nodes, visit all live operands. if (auto *PHI = dyn_cast<PHINode>(V)) { - assert(LivenessAA && - "Expected liveness in the presence of instructions!"); + LivenessInfo &LI = GetLivenessInfo(*PHI->getFunction()); for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) { BasicBlock *IncomingBB = PHI->getIncomingBlock(u); - if (LivenessAA->isEdgeDead(IncomingBB, PHI->getParent())) { - AnyDead = true; + if (LI.LivenessAA->isEdgeDead(IncomingBB, PHI->getParent())) { + LI.AnyDead = true; + UsedAssumedInformation |= !LI.LivenessAA->isAtFixpoint(); continue; } Worklist.push_back( @@ -355,9 +412,9 @@ static bool genericValueTraversal( } if (auto *Arg = dyn_cast<Argument>(V)) { - if (!Intraprocedural && !Arg->hasPassPointeeByValueCopyAttr()) { + if ((VS & AA::Interprocedural) && !Arg->hasPassPointeeByValueCopyAttr()) { SmallVector<Item> CallSiteValues; - bool AllCallSitesKnown = true; + bool UsedAssumedInformation = false; if (A.checkForAllCallSites( [&](AbstractCallSite ACS) { // Callbacks might not have a corresponding call site operand, @@ -368,7 +425,7 @@ static bool genericValueTraversal( CallSiteValues.push_back({CSOp, ACS.getInstruction()}); return true; }, - *Arg->getParent(), true, &QueryingAA, AllCallSitesKnown)) { + *Arg->getParent(), true, &QueryingAA, UsedAssumedInformation)) { Worklist.append(CallSiteValues); continue; } @@ -376,14 +433,13 @@ static bool genericValueTraversal( } if (UseValueSimplify && !isa<Constant>(V)) { - bool UsedAssumedInformation = false; Optional<Value *> SimpleV = A.getAssumedSimplified(*V, QueryingAA, UsedAssumedInformation); - if (!SimpleV.hasValue()) + if (!SimpleV) continue; Value *NewV = SimpleV.getValue(); if (NewV && NewV != V) { - if (!Intraprocedural || !CtxI || + if ((VS & AA::Interprocedural) || !CtxI || AA::isValidInScope(*NewV, CtxI->getFunction())) { Worklist.push_back({NewV, CtxI}); continue; @@ -391,6 +447,37 @@ static bool genericValueTraversal( } } + if (auto *LI = dyn_cast<LoadInst>(V)) { + bool UsedAssumedInformation = false; + // If we ask for the potentially loaded values from the initial pointer we + // will simply end up here again. The load is as far as we can make it. + if (LI->getPointerOperand() != InitialV) { + SmallSetVector<Value *, 4> PotentialCopies; + SmallSetVector<Instruction *, 4> PotentialValueOrigins; + if (AA::getPotentiallyLoadedValues(A, *LI, PotentialCopies, + PotentialValueOrigins, QueryingAA, + UsedAssumedInformation, + /* OnlyExact */ true)) { + // Values have to be dynamically unique or we loose the fact that a + // single llvm::Value might represent two runtime values (e.g., stack + // locations in different recursive calls). + bool DynamicallyUnique = + llvm::all_of(PotentialCopies, [&A, &QueryingAA](Value *PC) { + return AA::isDynamicallyUnique(A, QueryingAA, *PC); + }); + if (DynamicallyUnique && + ((VS & AA::Interprocedural) || !CtxI || + llvm::all_of(PotentialCopies, [CtxI](Value *PC) { + return AA::isValidInScope(*PC, CtxI->getFunction()); + }))) { + for (auto *PotentialCopy : PotentialCopies) + Worklist.push_back({PotentialCopy, CtxI}); + continue; + } + } + } + } + // Once a leaf is reached we inform the user through the callback. if (!VisitValueCB(*V, CtxI, State, Iteration > 1)) { LLVM_DEBUG(dbgs() << "Generic value traversal visit callback failed for: " @@ -400,8 +487,10 @@ static bool genericValueTraversal( } while (!Worklist.empty()); // If we actually used liveness information so we have to record a dependence. - if (AnyDead) - A.recordDependence(*LivenessAA, QueryingAA, DepClassTy::OPTIONAL); + for (auto &It : LivenessAAs) + if (It.second.AnyDead) + A.recordDependence(*It.second.LivenessAA, QueryingAA, + DepClassTy::OPTIONAL); // All values have been visited. return true; @@ -411,7 +500,8 @@ bool AA::getAssumedUnderlyingObjects(Attributor &A, const Value &Ptr, SmallVectorImpl<Value *> &Objects, const AbstractAttribute &QueryingAA, const Instruction *CtxI, - bool Intraprocedural) { + bool &UsedAssumedInformation, + AA::ValueScope VS) { auto StripCB = [&](Value *V) { return getUnderlyingObject(V); }; SmallPtrSet<Value *, 8> SeenObjects; auto VisitValueCB = [&SeenObjects](Value &Val, const Instruction *, @@ -423,15 +513,16 @@ bool AA::getAssumedUnderlyingObjects(Attributor &A, const Value &Ptr, }; if (!genericValueTraversal<decltype(Objects)>( A, IRPosition::value(Ptr), QueryingAA, Objects, VisitValueCB, CtxI, - true, 32, StripCB, Intraprocedural)) + UsedAssumedInformation, true, 32, StripCB, VS)) return false; return true; } -const Value *stripAndAccumulateMinimalOffsets( - Attributor &A, const AbstractAttribute &QueryingAA, const Value *Val, - const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, - bool UseAssumed = false) { +static const Value * +stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA, + const Value *Val, const DataLayout &DL, APInt &Offset, + bool GetMinOffset, bool AllowNonInbounds, + bool UseAssumed = false) { auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool { const IRPosition &Pos = IRPosition::value(V); @@ -442,14 +533,20 @@ const Value *stripAndAccumulateMinimalOffsets( : DepClassTy::NONE); ConstantRange Range = UseAssumed ? ValueConstantRangeAA.getAssumed() : ValueConstantRangeAA.getKnown(); + if (Range.isFullSet()) + return false; + // We can only use the lower part of the range because the upper part can // be higher than what the value can really be. - ROffset = Range.getSignedMin(); + if (GetMinOffset) + ROffset = Range.getSignedMin(); + else + ROffset = Range.getSignedMax(); return true; }; return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds, - /* AllowInvariant */ false, + /* AllowInvariant */ true, AttributorAnalysis); } @@ -458,8 +555,9 @@ getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Ptr, int64_t &BytesOffset, const DataLayout &DL, bool AllowNonInbounds = false) { APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); - const Value *Base = stripAndAccumulateMinimalOffsets( - A, QueryingAA, Ptr, DL, OffsetAPInt, AllowNonInbounds); + const Value *Base = + stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt, + /* GetMinOffset */ true, AllowNonInbounds); BytesOffset = OffsetAPInt.getSExtValue(); return Base; @@ -493,10 +591,9 @@ static void clampReturnedValueStates( LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV << " AA: " << AA.getAsStr() << " @ " << RVPos << "\n"); const StateType &AAS = AA.getState(); - if (T.hasValue()) - *T &= AAS; - else - T = AAS; + if (!T) + T = StateType::getBestState(AAS); + *T &= AAS; LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T << "\n"); return T->isValidState(); @@ -504,7 +601,7 @@ static void clampReturnedValueStates( if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA)) S.indicatePessimisticFixpoint(); - else if (T.hasValue()) + else if (T) S ^= *T; } @@ -560,20 +657,19 @@ static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA, LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction() << " AA: " << AA.getAsStr() << " @" << ACSArgPos << "\n"); const StateType &AAS = AA.getState(); - if (T.hasValue()) - *T &= AAS; - else - T = AAS; + if (!T) + T = StateType::getBestState(AAS); + *T &= AAS; LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T << "\n"); return T->isValidState(); }; - bool AllCallSitesKnown; + bool UsedAssumedInformation = false; if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true, - AllCallSitesKnown)) + UsedAssumedInformation)) S.indicatePessimisticFixpoint(); - else if (T.hasValue()) + else if (T) S ^= *T; } @@ -667,7 +763,6 @@ struct AACallSiteReturnedFromReturned : public BaseType { return clampStateAndIndicateChange(S, AA.getState()); } }; -} // namespace /// Helper function to accumulate uses. template <class AAType, typename StateType = typename AAType::StateType> @@ -779,6 +874,7 @@ static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S, S += ParentState; } } +} // namespace /// ------------------------ PointerInfo --------------------------------------- @@ -786,9 +882,6 @@ namespace llvm { namespace AA { namespace PointerInfo { -/// An access kind description as used by AAPointerInfo. -struct OffsetAndSize; - struct State; } // namespace PointerInfo @@ -806,7 +899,7 @@ struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> { /// Helper that allows OffsetAndSize as a key in a DenseMap. template <> -struct DenseMapInfo<AA::PointerInfo ::OffsetAndSize> +struct DenseMapInfo<AAPointerInfo ::OffsetAndSize> : DenseMapInfo<std::pair<int64_t, int64_t>> {}; /// Helper for AA::PointerInfo::Acccess DenseMap/Set usage ignoring everythign @@ -822,90 +915,15 @@ struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> { } // namespace llvm -/// Helper to represent an access offset and size, with logic to deal with -/// uncertainty and check for overlapping accesses. -struct AA::PointerInfo::OffsetAndSize : public std::pair<int64_t, int64_t> { - using BaseTy = std::pair<int64_t, int64_t>; - OffsetAndSize(int64_t Offset, int64_t Size) : BaseTy(Offset, Size) {} - OffsetAndSize(const BaseTy &P) : BaseTy(P) {} - int64_t getOffset() const { return first; } - int64_t getSize() const { return second; } - static OffsetAndSize getUnknown() { return OffsetAndSize(Unknown, Unknown); } - - /// Return true if offset or size are unknown. - bool offsetOrSizeAreUnknown() const { - return getOffset() == OffsetAndSize::Unknown || - getSize() == OffsetAndSize::Unknown; - } - - /// Return true if this offset and size pair might describe an address that - /// overlaps with \p OAS. - bool mayOverlap(const OffsetAndSize &OAS) const { - // Any unknown value and we are giving up -> overlap. - if (offsetOrSizeAreUnknown() || OAS.offsetOrSizeAreUnknown()) - return true; - - // Check if one offset point is in the other interval [offset, offset+size]. - return OAS.getOffset() + OAS.getSize() > getOffset() && - OAS.getOffset() < getOffset() + getSize(); - } - - /// Constant used to represent unknown offset or sizes. - static constexpr int64_t Unknown = 1 << 31; -}; - -/// Implementation of the DenseMapInfo. -/// -///{ -inline llvm::AccessAsInstructionInfo::Access -llvm::AccessAsInstructionInfo::getEmptyKey() { - return Access(Base::getEmptyKey(), nullptr, AAPointerInfo::AK_READ, nullptr); -} -inline llvm::AccessAsInstructionInfo::Access -llvm::AccessAsInstructionInfo::getTombstoneKey() { - return Access(Base::getTombstoneKey(), nullptr, AAPointerInfo::AK_READ, - nullptr); -} -unsigned llvm::AccessAsInstructionInfo::getHashValue( - const llvm::AccessAsInstructionInfo::Access &A) { - return Base::getHashValue(A.getRemoteInst()); -} -bool llvm::AccessAsInstructionInfo::isEqual( - const llvm::AccessAsInstructionInfo::Access &LHS, - const llvm::AccessAsInstructionInfo::Access &RHS) { - return LHS.getRemoteInst() == RHS.getRemoteInst(); -} -inline llvm::DenseMapInfo<AAPointerInfo::Access>::Access -llvm::DenseMapInfo<AAPointerInfo::Access>::getEmptyKey() { - return AAPointerInfo::Access(nullptr, nullptr, AAPointerInfo::AK_READ, - nullptr); -} -inline llvm::DenseMapInfo<AAPointerInfo::Access>::Access -llvm::DenseMapInfo<AAPointerInfo::Access>::getTombstoneKey() { - return AAPointerInfo::Access(nullptr, nullptr, AAPointerInfo::AK_WRITE, - nullptr); -} - -unsigned llvm::DenseMapInfo<AAPointerInfo::Access>::getHashValue( - const llvm::DenseMapInfo<AAPointerInfo::Access>::Access &A) { - return detail::combineHashValue( - DenseMapInfo<Instruction *>::getHashValue(A.getRemoteInst()), - (A.isWrittenValueYetUndetermined() - ? ~0 - : DenseMapInfo<Value *>::getHashValue(A.getWrittenValue()))) + - A.getKind(); -} - -bool llvm::DenseMapInfo<AAPointerInfo::Access>::isEqual( - const llvm::DenseMapInfo<AAPointerInfo::Access>::Access &LHS, - const llvm::DenseMapInfo<AAPointerInfo::Access>::Access &RHS) { - return LHS == RHS; -} -///} - /// A type to track pointer/struct usage and accesses for AAPointerInfo. struct AA::PointerInfo::State : public AbstractState { + ~State() { + // We do not delete the Accesses objects but need to destroy them still. + for (auto &It : AccessBins) + It.second->~Accesses(); + } + /// Return the best possible representable state. static State getBestState(const State &SIS) { return State(); } @@ -916,9 +934,10 @@ struct AA::PointerInfo::State : public AbstractState { return R; } - State() {} - State(const State &SIS) : AccessBins(SIS.AccessBins) {} - State(State &&SIS) : AccessBins(std::move(SIS.AccessBins)) {} + State() = default; + State(State &&SIS) : AccessBins(std::move(SIS.AccessBins)) { + SIS.AccessBins.clear(); + } const State &getAssumed() const { return *this; } @@ -967,15 +986,11 @@ struct AA::PointerInfo::State : public AbstractState { return false; auto &Accs = It->getSecond(); auto &RAccs = RIt->getSecond(); - if (Accs.size() != RAccs.size()) + if (Accs->size() != RAccs->size()) return false; - auto AccIt = Accs.begin(), RAccIt = RAccs.begin(), AccE = Accs.end(); - while (AccIt != AccE) { - if (*AccIt != *RAccIt) + for (const auto &ZipIt : llvm::zip(*Accs, *RAccs)) + if (std::get<0>(ZipIt) != std::get<1>(ZipIt)) return false; - ++AccIt; - ++RAccIt; - } ++It; ++RIt; } @@ -984,42 +999,88 @@ struct AA::PointerInfo::State : public AbstractState { bool operator!=(const State &R) const { return !(*this == R); } /// We store accesses in a set with the instruction as key. - using Accesses = DenseSet<AAPointerInfo::Access, AccessAsInstructionInfo>; + struct Accesses { + SmallVector<AAPointerInfo::Access, 4> Accesses; + DenseMap<const Instruction *, unsigned> Map; + + unsigned size() const { return Accesses.size(); } + + using vec_iterator = decltype(Accesses)::iterator; + vec_iterator begin() { return Accesses.begin(); } + vec_iterator end() { return Accesses.end(); } + + using iterator = decltype(Map)::const_iterator; + iterator find(AAPointerInfo::Access &Acc) { + return Map.find(Acc.getRemoteInst()); + } + iterator find_end() { return Map.end(); } + + AAPointerInfo::Access &get(iterator &It) { + return Accesses[It->getSecond()]; + } + + void insert(AAPointerInfo::Access &Acc) { + Map[Acc.getRemoteInst()] = Accesses.size(); + Accesses.push_back(Acc); + } + }; /// We store all accesses in bins denoted by their offset and size. - using AccessBinsTy = DenseMap<OffsetAndSize, Accesses>; + using AccessBinsTy = DenseMap<AAPointerInfo::OffsetAndSize, Accesses *>; AccessBinsTy::const_iterator begin() const { return AccessBins.begin(); } AccessBinsTy::const_iterator end() const { return AccessBins.end(); } protected: /// The bins with all the accesses for the associated pointer. - DenseMap<OffsetAndSize, Accesses> AccessBins; + AccessBinsTy AccessBins; /// Add a new access to the state at offset \p Offset and with size \p Size. /// The access is associated with \p I, writes \p Content (if anything), and /// is of kind \p Kind. /// \Returns CHANGED, if the state changed, UNCHANGED otherwise. - ChangeStatus addAccess(int64_t Offset, int64_t Size, Instruction &I, - Optional<Value *> Content, + ChangeStatus addAccess(Attributor &A, int64_t Offset, int64_t Size, + Instruction &I, Optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty, Instruction *RemoteI = nullptr, Accesses *BinPtr = nullptr) { - OffsetAndSize Key{Offset, Size}; - Accesses &Bin = BinPtr ? *BinPtr : AccessBins[Key]; + AAPointerInfo::OffsetAndSize Key{Offset, Size}; + Accesses *&Bin = BinPtr ? BinPtr : AccessBins[Key]; + if (!Bin) + Bin = new (A.Allocator) Accesses; AAPointerInfo::Access Acc(&I, RemoteI ? RemoteI : &I, Content, Kind, Ty); // Check if we have an access for this instruction in this bin, if not, // simply add it. - auto It = Bin.find(Acc); - if (It == Bin.end()) { - Bin.insert(Acc); + auto It = Bin->find(Acc); + if (It == Bin->find_end()) { + Bin->insert(Acc); return ChangeStatus::CHANGED; } // If the existing access is the same as then new one, nothing changed. - AAPointerInfo::Access Before = *It; + AAPointerInfo::Access &Current = Bin->get(It); + AAPointerInfo::Access Before = Current; // The new one will be combined with the existing one. - *It &= Acc; - return *It == Before ? ChangeStatus::UNCHANGED : ChangeStatus::CHANGED; + Current &= Acc; + return Current == Before ? ChangeStatus::UNCHANGED : ChangeStatus::CHANGED; + } + + /// See AAPointerInfo::forallInterferingAccesses. + bool forallInterferingAccesses( + AAPointerInfo::OffsetAndSize OAS, + function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const { + if (!isValidState()) + return false; + + for (auto &It : AccessBins) { + AAPointerInfo::OffsetAndSize ItOAS = It.getFirst(); + if (!OAS.mayOverlap(ItOAS)) + continue; + bool IsExact = OAS == ItOAS && !OAS.offsetOrSizeAreUnknown(); + for (auto &Access : *It.getSecond()) + if (!CB(Access, IsExact)) + return false; + } + return true; } /// See AAPointerInfo::forallInterferingAccesses. @@ -1028,10 +1089,11 @@ protected: function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const { if (!isValidState()) return false; + // First find the offset and size of I. - OffsetAndSize OAS(-1, -1); + AAPointerInfo::OffsetAndSize OAS(-1, -1); for (auto &It : AccessBins) { - for (auto &Access : It.getSecond()) { + for (auto &Access : *It.getSecond()) { if (Access.getRemoteInst() == &I) { OAS = It.getFirst(); break; @@ -1040,21 +1102,13 @@ protected: if (OAS.getSize() != -1) break; } + // No access for I was found, we are done. if (OAS.getSize() == -1) return true; // Now that we have an offset and size, find all overlapping ones and use // the callback on the accesses. - for (auto &It : AccessBins) { - OffsetAndSize ItOAS = It.getFirst(); - if (!OAS.mayOverlap(ItOAS)) - continue; - bool IsExact = OAS == ItOAS && !OAS.offsetOrSizeAreUnknown(); - for (auto &Access : It.getSecond()) - if (!CB(Access, IsExact)) - return false; - } - return true; + return forallInterferingAccesses(OAS, CB); } private: @@ -1062,6 +1116,7 @@ private: BooleanState BS; }; +namespace { struct AAPointerInfoImpl : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> { using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>; @@ -1084,22 +1139,18 @@ struct AAPointerInfoImpl } bool forallInterferingAccesses( - LoadInst &LI, function_ref<bool(const AAPointerInfo::Access &, bool)> CB) + OffsetAndSize OAS, + function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const override { - return State::forallInterferingAccesses(LI, CB); + return State::forallInterferingAccesses(OAS, CB); } bool forallInterferingAccesses( - StoreInst &SI, function_ref<bool(const AAPointerInfo::Access &, bool)> CB) - const override { - return State::forallInterferingAccesses(SI, CB); - } - bool forallInterferingWrites( - Attributor &A, const AbstractAttribute &QueryingAA, LoadInst &LI, + Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I, function_ref<bool(const Access &, bool)> UserCB) const override { SmallPtrSet<const Access *, 8> DominatingWrites; - SmallVector<std::pair<const Access *, bool>, 8> InterferingWrites; + SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses; - Function &Scope = *LI.getFunction(); + Function &Scope = *I.getFunction(); const auto &NoSyncAA = A.getAAFor<AANoSync>( QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL); const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>( @@ -1127,13 +1178,15 @@ struct AAPointerInfoImpl // TODO: Use inter-procedural reachability and dominance. const auto &NoRecurseAA = A.getAAFor<AANoRecurse>( - QueryingAA, IRPosition::function(*LI.getFunction()), - DepClassTy::OPTIONAL); + QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL); - const bool CanUseCFGResoning = CanIgnoreThreading(LI); + const bool FindInterferingWrites = I.mayReadFromMemory(); + const bool FindInterferingReads = I.mayWriteToMemory(); + const bool UseDominanceReasoning = FindInterferingWrites; + const bool CanUseCFGResoning = CanIgnoreThreading(I); InformationCache &InfoCache = A.getInfoCache(); const DominatorTree *DT = - NoRecurseAA.isKnownNoRecurse() + NoRecurseAA.isKnownNoRecurse() && UseDominanceReasoning ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>( Scope) : nullptr; @@ -1189,33 +1242,37 @@ struct AAPointerInfoImpl } auto AccessCB = [&](const Access &Acc, bool Exact) { - if (!Acc.isWrite()) + if ((!FindInterferingWrites || !Acc.isWrite()) && + (!FindInterferingReads || !Acc.isRead())) return true; // For now we only filter accesses based on CFG reasoning which does not // work yet if we have threading effects, or the access is complicated. if (CanUseCFGResoning) { - if (!AA::isPotentiallyReachable(A, *Acc.getLocalInst(), LI, QueryingAA, - IsLiveInCalleeCB)) + if ((!Acc.isWrite() || + !AA::isPotentiallyReachable(A, *Acc.getLocalInst(), I, QueryingAA, + IsLiveInCalleeCB)) && + (!Acc.isRead() || + !AA::isPotentiallyReachable(A, I, *Acc.getLocalInst(), QueryingAA, + IsLiveInCalleeCB))) return true; - if (DT && Exact && - (Acc.getLocalInst()->getFunction() == LI.getFunction()) && + if (DT && Exact && (Acc.getLocalInst()->getFunction() == &Scope) && IsSameThreadAsLoad(Acc)) { - if (DT->dominates(Acc.getLocalInst(), &LI)) + if (DT->dominates(Acc.getLocalInst(), &I)) DominatingWrites.insert(&Acc); } } - InterferingWrites.push_back({&Acc, Exact}); + InterferingAccesses.push_back({&Acc, Exact}); return true; }; - if (!State::forallInterferingAccesses(LI, AccessCB)) + if (!State::forallInterferingAccesses(I, AccessCB)) return false; // If we cannot use CFG reasoning we only filter the non-write accesses // and are done here. if (!CanUseCFGResoning) { - for (auto &It : InterferingWrites) + for (auto &It : InterferingAccesses) if (!UserCB(*It.first, It.second)) return false; return true; @@ -1242,47 +1299,52 @@ struct AAPointerInfoImpl return false; }; - // Run the user callback on all writes we cannot skip and return if that + // Run the user callback on all accesses we cannot skip and return if that // succeeded for all or not. - unsigned NumInterferingWrites = InterferingWrites.size(); - for (auto &It : InterferingWrites) - if (!DT || NumInterferingWrites > MaxInterferingWrites || - !CanSkipAccess(*It.first, It.second)) + unsigned NumInterferingAccesses = InterferingAccesses.size(); + for (auto &It : InterferingAccesses) { + if (!DT || NumInterferingAccesses > MaxInterferingAccesses || + !CanSkipAccess(*It.first, It.second)) { if (!UserCB(*It.first, It.second)) return false; + } + } return true; } - ChangeStatus translateAndAddCalleeState(Attributor &A, - const AAPointerInfo &CalleeAA, - int64_t CallArgOffset, CallBase &CB) { + ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA, + int64_t Offset, CallBase &CB, + bool FromCallee = false) { using namespace AA::PointerInfo; - if (!CalleeAA.getState().isValidState() || !isValidState()) + if (!OtherAA.getState().isValidState() || !isValidState()) return indicatePessimisticFixpoint(); - const auto &CalleeImplAA = static_cast<const AAPointerInfoImpl &>(CalleeAA); - bool IsByval = CalleeImplAA.getAssociatedArgument()->hasByValAttr(); + const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA); + bool IsByval = + FromCallee && OtherAAImpl.getAssociatedArgument()->hasByValAttr(); // Combine the accesses bin by bin. ChangeStatus Changed = ChangeStatus::UNCHANGED; - for (auto &It : CalleeImplAA.getState()) { + for (auto &It : OtherAAImpl.getState()) { OffsetAndSize OAS = OffsetAndSize::getUnknown(); - if (CallArgOffset != OffsetAndSize::Unknown) - OAS = OffsetAndSize(It.first.getOffset() + CallArgOffset, - It.first.getSize()); - Accesses &Bin = AccessBins[OAS]; - for (const AAPointerInfo::Access &RAcc : It.second) { + if (Offset != OffsetAndSize::Unknown) + OAS = OffsetAndSize(It.first.getOffset() + Offset, It.first.getSize()); + Accesses *Bin = AccessBins.lookup(OAS); + for (const AAPointerInfo::Access &RAcc : *It.second) { if (IsByval && !RAcc.isRead()) continue; bool UsedAssumedInformation = false; - Optional<Value *> Content = A.translateArgumentToCallSiteContent( - RAcc.getContent(), CB, *this, UsedAssumedInformation); - AccessKind AK = - AccessKind(RAcc.getKind() & (IsByval ? AccessKind::AK_READ - : AccessKind::AK_READ_WRITE)); + AccessKind AK = RAcc.getKind(); + Optional<Value *> Content = RAcc.getContent(); + if (FromCallee) { + Content = A.translateArgumentToCallSiteContent( + RAcc.getContent(), CB, *this, UsedAssumedInformation); + AK = AccessKind( + AK & (IsByval ? AccessKind::AK_READ : AccessKind::AK_READ_WRITE)); + } Changed = - Changed | addAccess(OAS.getOffset(), OAS.getSize(), CB, Content, AK, - RAcc.getType(), RAcc.getRemoteInst(), &Bin); + Changed | addAccess(A, OAS.getOffset(), OAS.getSize(), CB, Content, + AK, RAcc.getType(), RAcc.getRemoteInst(), Bin); } } return Changed; @@ -1305,7 +1367,7 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { bool handleAccess(Attributor &A, Instruction &I, Value &Ptr, Optional<Value *> Content, AccessKind Kind, int64_t Offset, ChangeStatus &Changed, Type *Ty, - int64_t Size = AA::PointerInfo::OffsetAndSize::Unknown) { + int64_t Size = OffsetAndSize::Unknown) { using namespace AA::PointerInfo; // No need to find a size if one is given or the offset is unknown. if (Offset != OffsetAndSize::Unknown && Size == OffsetAndSize::Unknown && @@ -1315,13 +1377,13 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { if (!AccessSize.isScalable()) Size = AccessSize.getFixedSize(); } - Changed = Changed | addAccess(Offset, Size, I, Content, Kind, Ty); + Changed = Changed | addAccess(A, Offset, Size, I, Content, Kind, Ty); return true; }; /// Helper struct, will support ranges eventually. struct OffsetInfo { - int64_t Offset = AA::PointerInfo::OffsetAndSize::Unknown; + int64_t Offset = OffsetAndSize::Unknown; bool operator==(const OffsetInfo &OI) const { return Offset == OI.Offset; } }; @@ -1329,7 +1391,6 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { /// See AbstractAttribute::updateImpl(...). ChangeStatus updateImpl(Attributor &A) override { using namespace AA::PointerInfo; - State S = getState(); ChangeStatus Changed = ChangeStatus::UNCHANGED; Value &AssociatedValue = getAssociatedValue(); @@ -1337,7 +1398,7 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { DenseMap<Value *, OffsetInfo> OffsetInfoMap; OffsetInfoMap[&AssociatedValue] = OffsetInfo{0}; - auto HandlePassthroughUser = [&](Value *Usr, OffsetInfo &PtrOI, + auto HandlePassthroughUser = [&](Value *Usr, OffsetInfo PtrOI, bool &Follow) { OffsetInfo &UsrOI = OffsetInfoMap[Usr]; UsrOI = PtrOI; @@ -1475,8 +1536,8 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { const auto &CSArgPI = A.getAAFor<AAPointerInfo>( *this, IRPosition::callsite_argument(*CB, ArgNo), DepClassTy::REQUIRED); - Changed = translateAndAddCalleeState( - A, CSArgPI, OffsetInfoMap[CurPtr].Offset, *CB) | + Changed = translateAndAddState(A, CSArgPI, + OffsetInfoMap[CurPtr].Offset, *CB) | Changed; return true; } @@ -1497,7 +1558,7 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { }; if (!A.checkForAllUses(UsePred, *this, AssociatedValue, /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL, - EquivalentUseCB)) + /* IgnoreDroppableUses */ true, EquivalentUseCB)) return indicatePessimisticFixpoint(); LLVM_DEBUG({ @@ -1505,15 +1566,19 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl { for (auto &It : AccessBins) { dbgs() << "[" << It.first.getOffset() << "-" << It.first.getOffset() + It.first.getSize() - << "] : " << It.getSecond().size() << "\n"; - for (auto &Acc : It.getSecond()) { + << "] : " << It.getSecond()->size() << "\n"; + for (auto &Acc : *It.getSecond()) { dbgs() << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n"; if (Acc.getLocalInst() != Acc.getRemoteInst()) dbgs() << " --> " << *Acc.getRemoteInst() << "\n"; - if (!Acc.isWrittenValueYetUndetermined()) - dbgs() << " - " << Acc.getWrittenValue() << "\n"; + if (!Acc.isWrittenValueYetUndetermined()) { + if (Acc.getWrittenValue()) + dbgs() << " - c: " << *Acc.getWrittenValue() << "\n"; + else + dbgs() << " - c: <unknown>\n"; + } } } }); @@ -1576,7 +1641,7 @@ struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating { LengthVal = Length->getSExtValue(); Value &Ptr = getAssociatedValue(); unsigned ArgNo = getIRPosition().getCallSiteArgNo(); - ChangeStatus Changed; + ChangeStatus Changed = ChangeStatus::UNCHANGED; if (ArgNo == 0) { handleAccess(A, *MI, Ptr, nullptr, AccessKind::AK_WRITE, 0, Changed, nullptr, LengthVal); @@ -1601,7 +1666,8 @@ struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating { const IRPosition &ArgPos = IRPosition::argument(*Arg); auto &ArgAA = A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED); - return translateAndAddCalleeState(A, ArgAA, 0, *cast<CallBase>(getCtxI())); + return translateAndAddState(A, ArgAA, 0, *cast<CallBase>(getCtxI()), + /* FromCallee */ true); } /// See AbstractAttribute::trackStatistics() @@ -1619,9 +1685,11 @@ struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating { AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition()); } }; +} // namespace /// -----------------------NoUnwind Function Attribute-------------------------- +namespace { struct AANoUnwindImpl : AANoUnwind { AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {} @@ -1693,9 +1761,11 @@ struct AANoUnwindCallSite final : AANoUnwindImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); } }; +} // namespace /// --------------------- Function Return Values ------------------------------- +namespace { /// "Attribute" that collects all potential returned values and the return /// instructions that they arise from. /// @@ -1821,7 +1891,7 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) { // Check if we have an assumed unique return value that we could manifest. Optional<Value *> UniqueRV = getAssumedUniqueReturnValue(A); - if (!UniqueRV.hasValue() || !UniqueRV.getValue()) + if (!UniqueRV || !UniqueRV.getValue()) return Changed; // Bookkeeping. @@ -1893,17 +1963,18 @@ ChangeStatus AAReturnedValuesImpl::updateImpl(Attributor &A) { return true; }; + bool UsedAssumedInformation = false; auto ReturnInstCB = [&](Instruction &I) { ReturnInst &Ret = cast<ReturnInst>(I); return genericValueTraversal<ReturnInst>( A, IRPosition::value(*Ret.getReturnValue()), *this, Ret, ReturnValueCB, - &I, /* UseValueSimplify */ true, /* MaxValues */ 16, - /* StripCB */ nullptr, /* Intraprocedural */ true); + &I, UsedAssumedInformation, /* UseValueSimplify */ true, + /* MaxValues */ 16, + /* StripCB */ nullptr, AA::Intraprocedural); }; // Discover returned values from all live returned instructions in the // associated function. - bool UsedAssumedInformation = false; if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret}, UsedAssumedInformation)) return indicatePessimisticFixpoint(); @@ -1941,20 +2012,10 @@ struct AAReturnedValuesCallSite final : AAReturnedValuesImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override {} }; +} // namespace /// ------------------------ NoSync Function Attribute ------------------------- -struct AANoSyncImpl : AANoSync { - AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {} - - const std::string getAsStr() const override { - return getAssumed() ? "nosync" : "may-sync"; - } - - /// See AbstractAttribute::updateImpl(...). - ChangeStatus updateImpl(Attributor &A) override; -}; - bool AANoSync::isNonRelaxedAtomic(const Instruction *I) { if (!I->isAtomic()) return false; @@ -1997,6 +2058,18 @@ bool AANoSync::isNoSyncIntrinsic(const Instruction *I) { return false; } +namespace { +struct AANoSyncImpl : AANoSync { + AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {} + + const std::string getAsStr() const override { + return getAssumed() ? "nosync" : "may-sync"; + } + + /// See AbstractAttribute::updateImpl(...). + ChangeStatus updateImpl(Attributor &A) override; +}; + ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) { auto CheckRWInstForNoSync = [&](Instruction &I) { @@ -2059,9 +2132,11 @@ struct AANoSyncCallSite final : AANoSyncImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); } }; +} // namespace /// ------------------------ No-Free Attributes ---------------------------- +namespace { struct AANoFreeImpl : public AANoFree { AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {} @@ -2243,8 +2318,10 @@ struct AANoFreeCallSiteReturned final : AANoFreeFloating { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) } }; +} // namespace /// ------------------------ NonNull Argument Attribute ------------------------ +namespace { static int64_t getKnownNonNullAndDerefBytesForUse( Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue, const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) { @@ -2332,7 +2409,7 @@ struct AANonNullImpl : AANonNull { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { - Value &V = getAssociatedValue(); + Value &V = *getAssociatedValue().stripPointerCasts(); if (!NullIsDefined && hasAttr({Attribute::NonNull, Attribute::Dereferenceable}, /* IgnoreSubsumingPositions */ false, &A)) { @@ -2356,7 +2433,7 @@ struct AANonNullImpl : AANonNull { } } - if (isa<GlobalValue>(&getAssociatedValue())) { + if (isa<GlobalValue>(V)) { indicatePessimisticFixpoint(); return; } @@ -2419,8 +2496,10 @@ struct AANonNullFloating : public AANonNullImpl { }; StateType T; + bool UsedAssumedInformation = false; if (!genericValueTraversal<StateType>(A, getIRPosition(), *this, T, - VisitValueCB, getCtxI())) + VisitValueCB, getCtxI(), + UsedAssumedInformation)) return indicatePessimisticFixpoint(); return clampStateAndIndicateChange(getState(), T); @@ -2472,9 +2551,11 @@ struct AANonNullCallSiteReturned final /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) } }; +} // namespace /// ------------------------ No-Recurse Attributes ---------------------------- +namespace { struct AANoRecurseImpl : public AANoRecurse { AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {} @@ -2498,14 +2579,15 @@ struct AANoRecurseFunction final : AANoRecurseImpl { DepClassTy::NONE); return NoRecurseAA.isKnownNoRecurse(); }; - bool AllCallSitesKnown; - if (A.checkForAllCallSites(CallSitePred, *this, true, AllCallSitesKnown)) { + bool UsedAssumedInformation = false; + if (A.checkForAllCallSites(CallSitePred, *this, true, + UsedAssumedInformation)) { // If we know all call sites and all are known no-recurse, we are done. // If all known call sites, which might not be all that exist, are known // to be no-recurse, we are not done but we can continue to assume // no-recurse. If one of the call sites we have not visited will become // live, another update is triggered. - if (AllCallSitesKnown) + if (!UsedAssumedInformation) indicateOptimisticFixpoint(); return ChangeStatus::UNCHANGED; } @@ -2549,9 +2631,11 @@ struct AANoRecurseCallSite final : AANoRecurseImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); } }; +} // namespace /// -------------------- Undefined-Behavior Attributes ------------------------ +namespace { struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A) : AAUndefinedBehavior(IRP, A) {} @@ -2582,7 +2666,7 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { // Either we stopped and the appropriate action was taken, // or we got back a simplified value to continue. Optional<Value *> SimplifiedPtrOp = stopOnUndefOrAssumed(A, PtrOp, &I); - if (!SimplifiedPtrOp.hasValue() || !SimplifiedPtrOp.getValue()) + if (!SimplifiedPtrOp || !SimplifiedPtrOp.getValue()) return true; const Value *PtrOpVal = SimplifiedPtrOp.getValue(); @@ -2627,7 +2711,7 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { // or we got back a simplified value to continue. Optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(A, BrInst->getCondition(), BrInst); - if (!SimplifiedCond.hasValue() || !SimplifiedCond.getValue()) + if (!SimplifiedCond || !*SimplifiedCond) return true; AssumedNoUBInsts.insert(&I); return true; @@ -2673,10 +2757,9 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { IRPosition::value(*ArgVal), *this, UsedAssumedInformation); if (UsedAssumedInformation) continue; - if (SimplifiedVal.hasValue() && !SimplifiedVal.getValue()) + if (SimplifiedVal && !SimplifiedVal.getValue()) return true; - if (!SimplifiedVal.hasValue() || - isa<UndefValue>(*SimplifiedVal.getValue())) { + if (!SimplifiedVal || isa<UndefValue>(*SimplifiedVal.getValue())) { KnownUBInsts.insert(&I); continue; } @@ -2691,40 +2774,38 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { return true; }; - auto InspectReturnInstForUB = - [&](Value &V, const SmallSetVector<ReturnInst *, 4> RetInsts) { - // Check if a return instruction always cause UB or not - // Note: It is guaranteed that the returned position of the anchor - // scope has noundef attribute when this is called. - // We also ensure the return position is not "assumed dead" - // because the returned value was then potentially simplified to - // `undef` in AAReturnedValues without removing the `noundef` - // attribute yet. + auto InspectReturnInstForUB = [&](Instruction &I) { + auto &RI = cast<ReturnInst>(I); + // Either we stopped and the appropriate action was taken, + // or we got back a simplified return value to continue. + Optional<Value *> SimplifiedRetValue = + stopOnUndefOrAssumed(A, RI.getReturnValue(), &I); + if (!SimplifiedRetValue || !*SimplifiedRetValue) + return true; - // When the returned position has noundef attriubte, UB occur in the - // following cases. - // (1) Returned value is known to be undef. - // (2) The value is known to be a null pointer and the returned - // position has nonnull attribute (because the returned value is - // poison). - bool FoundUB = false; - if (isa<UndefValue>(V)) { - FoundUB = true; - } else { - if (isa<ConstantPointerNull>(V)) { - auto &NonNullAA = A.getAAFor<AANonNull>( - *this, IRPosition::returned(*getAnchorScope()), - DepClassTy::NONE); - if (NonNullAA.isKnownNonNull()) - FoundUB = true; - } - } + // Check if a return instruction always cause UB or not + // Note: It is guaranteed that the returned position of the anchor + // scope has noundef attribute when this is called. + // We also ensure the return position is not "assumed dead" + // because the returned value was then potentially simplified to + // `undef` in AAReturnedValues without removing the `noundef` + // attribute yet. - if (FoundUB) - for (ReturnInst *RI : RetInsts) - KnownUBInsts.insert(RI); - return true; - }; + // When the returned position has noundef attriubte, UB occurs in the + // following cases. + // (1) Returned value is known to be undef. + // (2) The value is known to be a null pointer and the returned + // position has nonnull attribute (because the returned value is + // poison). + if (isa<ConstantPointerNull>(*SimplifiedRetValue)) { + auto &NonNullAA = A.getAAFor<AANonNull>( + *this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE); + if (NonNullAA.isKnownNonNull()) + KnownUBInsts.insert(&I); + } + + return true; + }; bool UsedAssumedInformation = false; A.checkForAllInstructions(InspectMemAccessInstForUB, *this, @@ -2747,8 +2828,9 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { auto &RetPosNoUndefAA = A.getAAFor<AANoUndef>(*this, ReturnIRP, DepClassTy::NONE); if (RetPosNoUndefAA.isKnownNoUndef()) - A.checkForAllReturnedValuesAndReturnInsts(InspectReturnInstForUB, - *this); + A.checkForAllInstructions(InspectReturnInstForUB, *this, + {Instruction::Ret}, UsedAssumedInformation, + /* CheckBBLivenessOnly */ true); } } @@ -2776,7 +2858,7 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { case Instruction::AtomicRMW: return !AssumedNoUBInsts.count(I); case Instruction::Br: { - auto BrInst = cast<BranchInst>(I); + auto *BrInst = cast<BranchInst>(I); if (BrInst->isUnconditional()) return false; return !AssumedNoUBInsts.count(I); @@ -2847,13 +2929,13 @@ private: IRPosition::value(*V), *this, UsedAssumedInformation); if (!UsedAssumedInformation) { // Don't depend on assumed values. - if (!SimplifiedV.hasValue()) { + if (!SimplifiedV) { // If it is known (which we tested above) but it doesn't have a value, // then we can assume `undef` and hence the instruction is UB. KnownUBInsts.insert(I); return llvm::None; } - if (!SimplifiedV.getValue()) + if (!*SimplifiedV) return nullptr; V = *SimplifiedV; } @@ -2877,9 +2959,11 @@ struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl { KnownUBInsts.size(); } }; +} // namespace /// ------------------------ Will-Return Attributes ---------------------------- +namespace { // Helper function that checks whether a function has any cycle which we don't // know if it is bounded or not. // Loops with maximum trip count are considered bounded, any other cycle not. @@ -3018,9 +3102,11 @@ struct AAWillReturnCallSite final : AAWillReturnImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); } }; +} // namespace /// -------------------AAReachability Attribute-------------------------- +namespace { struct AAReachabilityImpl : AAReachability { AAReachabilityImpl(const IRPosition &IRP, Attributor &A) : AAReachability(IRP, A) {} @@ -3032,10 +3118,6 @@ struct AAReachabilityImpl : AAReachability { /// See AbstractAttribute::updateImpl(...). ChangeStatus updateImpl(Attributor &A) override { - const auto &NoRecurseAA = A.getAAFor<AANoRecurse>( - *this, IRPosition::function(*getAnchorScope()), DepClassTy::REQUIRED); - if (!NoRecurseAA.isAssumedNoRecurse()) - return indicatePessimisticFixpoint(); return ChangeStatus::UNCHANGED; } }; @@ -3047,9 +3129,11 @@ struct AAReachabilityFunction final : public AAReachabilityImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(reachable); } }; +} // namespace /// ------------------------ NoAlias Argument Attribute ------------------------ +namespace { struct AANoAliasImpl : AANoAlias { AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) { assert(getAssociatedType()->isPointerTy() && @@ -3146,10 +3230,10 @@ struct AANoAliasArgument final // If the argument is never passed through callbacks, no-alias cannot break // synchronization. - bool AllCallSitesKnown; + bool UsedAssumedInformation = false; if (A.checkForAllCallSites( [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this, - true, AllCallSitesKnown)) + true, UsedAssumedInformation)) return Base::updateImpl(A); // TODO: add no-alias but make sure it doesn't break synchronization by @@ -3246,14 +3330,20 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl { return false; } + auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) { + const auto &DerefAA = A.getAAFor<AADereferenceable>( + *this, IRPosition::value(*O), DepClassTy::OPTIONAL); + return DerefAA.getAssumedDereferenceableBytes(); + }; + A.recordDependence(NoAliasAA, *this, DepClassTy::OPTIONAL); const IRPosition &VIRP = IRPosition::value(getAssociatedValue()); const Function *ScopeFn = VIRP.getAnchorScope(); auto &NoCaptureAA = A.getAAFor<AANoCapture>(*this, VIRP, DepClassTy::NONE); // Check whether the value is captured in the scope using AANoCapture. - // Look at CFG and check only uses possibly executed before this - // callsite. + // Look at CFG and check only uses possibly executed before this + // callsite. auto UsePred = [&](const Use &U, bool &Follow) -> bool { Instruction *UserI = cast<Instruction>(U.getUser()); @@ -3265,12 +3355,6 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl { return true; if (ScopeFn) { - const auto &ReachabilityAA = A.getAAFor<AAReachability>( - *this, IRPosition::function(*ScopeFn), DepClassTy::OPTIONAL); - - if (!ReachabilityAA.isAssumedReachable(A, *UserI, *getCtxI())) - return true; - if (auto *CB = dyn_cast<CallBase>(UserI)) { if (CB->isArgOperand(&U)) { @@ -3284,17 +3368,26 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl { return true; } } + + if (!AA::isPotentiallyReachable(A, *UserI, *getCtxI(), *this)) + return true; } - // For cases which can potentially have more users - if (isa<GetElementPtrInst>(U) || isa<BitCastInst>(U) || isa<PHINode>(U) || - isa<SelectInst>(U)) { + // TODO: We should track the capturing uses in AANoCapture but the problem + // is CGSCC runs. For those we would need to "allow" AANoCapture for + // a value in the module slice. + switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) { + case UseCaptureKind::NO_CAPTURE: + return true; + case UseCaptureKind::MAY_CAPTURE: + LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI + << "\n"); + return false; + case UseCaptureKind::PASSTHROUGH: Follow = true; return true; } - - LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *U << "\n"); - return false; + llvm_unreachable("unknown UseCaptureKind"); }; if (!NoCaptureAA.isAssumedNoCaptureMaybeReturned()) { @@ -3423,12 +3516,21 @@ struct AANoAliasCallSiteReturned final : AANoAliasImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); } }; +} // namespace /// -------------------AAIsDead Function Attribute----------------------- +namespace { struct AAIsDeadValueImpl : public AAIsDead { AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {} + /// See AbstractAttribute::initialize(...). + void initialize(Attributor &A) override { + if (auto *Scope = getAnchorScope()) + if (!A.isRunOn(*Scope)) + indicatePessimisticFixpoint(); + } + /// See AAIsDead::isAssumedDead(). bool isAssumedDead() const override { return isAssumed(IS_DEAD); } @@ -3452,22 +3554,25 @@ struct AAIsDeadValueImpl : public AAIsDead { } /// See AbstractAttribute::getAsStr(). - const std::string getAsStr() const override { + virtual const std::string getAsStr() const override { return isAssumedDead() ? "assumed-dead" : "assumed-live"; } /// Check if all uses are assumed dead. bool areAllUsesAssumedDead(Attributor &A, Value &V) { // Callers might not check the type, void has no uses. - if (V.getType()->isVoidTy()) + if (V.getType()->isVoidTy() || V.use_empty()) return true; // If we replace a value with a constant there are no uses left afterwards. if (!isa<Constant>(V)) { + if (auto *I = dyn_cast<Instruction>(&V)) + if (!A.isRunOn(*I->getFunction())) + return false; bool UsedAssumedInformation = false; Optional<Constant *> C = A.getAssumedConstant(V, *this, UsedAssumedInformation); - if (!C.hasValue() || *C) + if (!C || *C) return true; } @@ -3477,7 +3582,8 @@ struct AAIsDeadValueImpl : public AAIsDead { // without going through N update cycles. This is not required for // correctness. return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false, - DepClassTy::REQUIRED); + DepClassTy::REQUIRED, + /* IgnoreDroppableUses */ false); } /// Determine if \p I is assumed to be side-effect free. @@ -3508,6 +3614,8 @@ struct AAIsDeadFloating : public AAIsDeadValueImpl { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { + AAIsDeadValueImpl::initialize(A); + if (isa<UndefValue>(getAssociatedValue())) { indicatePessimisticFixpoint(); return; @@ -3538,6 +3646,15 @@ struct AAIsDeadFloating : public AAIsDeadValueImpl { }); } + /// See AbstractAttribute::getAsStr(). + const std::string getAsStr() const override { + Instruction *I = dyn_cast<Instruction>(&getAssociatedValue()); + if (isa_and_nonnull<StoreInst>(I)) + if (isValidState()) + return "assumed-dead-store"; + return AAIsDeadValueImpl::getAsStr(); + } + /// See AbstractAttribute::updateImpl(...). ChangeStatus updateImpl(Attributor &A) override { Instruction *I = dyn_cast<Instruction>(&getAssociatedValue()); @@ -3553,6 +3670,10 @@ struct AAIsDeadFloating : public AAIsDeadValueImpl { return ChangeStatus::UNCHANGED; } + bool isRemovableStore() const override { + return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue()); + } + /// See AbstractAttribute::manifest(...). ChangeStatus manifest(Attributor &A) override { Value &V = getAssociatedValue(); @@ -3567,21 +3688,7 @@ struct AAIsDeadFloating : public AAIsDeadValueImpl { return ChangeStatus::CHANGED; } } - if (V.use_empty()) - return ChangeStatus::UNCHANGED; - - bool UsedAssumedInformation = false; - Optional<Constant *> C = - A.getAssumedConstant(V, *this, UsedAssumedInformation); - if (C.hasValue() && C.getValue()) - return ChangeStatus::UNCHANGED; - - // Replace the value with undef as it is dead but keep droppable uses around - // as they provide information we don't want to give up on just yet. - UndefValue &UV = *UndefValue::get(V.getType()); - bool AnyChange = - A.changeValueAfterManifest(V, UV, /* ChangeDropppable */ false); - return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; + return ChangeStatus::UNCHANGED; } /// See AbstractAttribute::trackStatistics() @@ -3596,23 +3703,22 @@ struct AAIsDeadArgument : public AAIsDeadFloating { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { + AAIsDeadFloating::initialize(A); if (!A.isFunctionIPOAmendable(*getAnchorScope())) indicatePessimisticFixpoint(); } /// See AbstractAttribute::manifest(...). ChangeStatus manifest(Attributor &A) override { - ChangeStatus Changed = AAIsDeadFloating::manifest(A); Argument &Arg = *getAssociatedArgument(); if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {})) if (A.registerFunctionSignatureRewrite( Arg, /* ReplacementTypes */ {}, Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{}, Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) { - Arg.dropDroppableUses(); return ChangeStatus::CHANGED; } - return Changed; + return ChangeStatus::UNCHANGED; } /// See AbstractAttribute::trackStatistics() @@ -3625,6 +3731,7 @@ struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { + AAIsDeadValueImpl::initialize(A); if (isa<UndefValue>(getAssociatedValue())) indicatePessimisticFixpoint(); } @@ -3661,7 +3768,7 @@ struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl { struct AAIsDeadCallSiteReturned : public AAIsDeadFloating { AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A) - : AAIsDeadFloating(IRP, A), IsAssumedSideEffectFree(true) {} + : AAIsDeadFloating(IRP, A) {} /// See AAIsDead::isAssumedDead(). bool isAssumedDead() const override { @@ -3670,6 +3777,7 @@ struct AAIsDeadCallSiteReturned : public AAIsDeadFloating { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { + AAIsDeadFloating::initialize(A); if (isa<UndefValue>(getAssociatedValue())) { indicatePessimisticFixpoint(); return; @@ -3707,7 +3815,7 @@ struct AAIsDeadCallSiteReturned : public AAIsDeadFloating { } private: - bool IsAssumedSideEffectFree; + bool IsAssumedSideEffectFree = true; }; struct AAIsDeadReturned : public AAIsDeadValueImpl { @@ -3727,9 +3835,8 @@ struct AAIsDeadReturned : public AAIsDeadValueImpl { return areAllUsesAssumedDead(A, *ACS.getInstruction()); }; - bool AllCallSitesKnown; if (!A.checkForAllCallSites(PredForCallSite, *this, true, - AllCallSitesKnown)) + UsedAssumedInformation)) return indicatePessimisticFixpoint(); return ChangeStatus::UNCHANGED; @@ -3761,17 +3868,13 @@ struct AAIsDeadFunction : public AAIsDead { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { - const Function *F = getAnchorScope(); - if (F && !F->isDeclaration()) { - // We only want to compute liveness once. If the function is not part of - // the SCC, skip it. - if (A.isRunOn(*const_cast<Function *>(F))) { - ToBeExploredFrom.insert(&F->getEntryBlock().front()); - assumeLive(A, F->getEntryBlock()); - } else { - indicatePessimisticFixpoint(); - } + Function *F = getAnchorScope(); + if (!F || F->isDeclaration() || !A.isRunOn(*F)) { + indicatePessimisticFixpoint(); + return; } + ToBeExploredFrom.insert(&F->getEntryBlock().front()); + assumeLive(A, F->getEntryBlock()); } /// See AbstractAttribute::getAsStr(). @@ -3834,6 +3937,9 @@ struct AAIsDeadFunction : public AAIsDead { ChangeStatus updateImpl(Attributor &A) override; bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override { + assert(From->getParent() == getAnchorScope() && + To->getParent() == getAnchorScope() && + "Used AAIsDead of the wrong function"); return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To)); } @@ -3973,7 +4079,7 @@ identifyAliveSuccessors(Attributor &A, const BranchInst &BI, } else { Optional<Constant *> C = A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation); - if (!C.hasValue() || isa_and_nonnull<UndefValue>(C.getValue())) { + if (!C || isa_and_nonnull<UndefValue>(*C)) { // No value yet, assume both edges are dead. } else if (isa_and_nonnull<ConstantInt>(*C)) { const BasicBlock *SuccBB = @@ -3995,7 +4101,7 @@ identifyAliveSuccessors(Attributor &A, const SwitchInst &SI, bool UsedAssumedInformation = false; Optional<Constant *> C = A.getAssumedConstant(*SI.getCondition(), AA, UsedAssumedInformation); - if (!C.hasValue() || isa_and_nonnull<UndefValue>(C.getValue())) { + if (!C || isa_and_nonnull<UndefValue>(C.getValue())) { // No value yet, assume all edges are dead. } else if (isa_and_nonnull<ConstantInt>(C.getValue())) { for (auto &CaseIt : SI.cases()) { @@ -4142,9 +4248,11 @@ struct AAIsDeadCallSite final : AAIsDeadFunction { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override {} }; +} // namespace /// -------------------- Dereferenceable Argument Attribute -------------------- +namespace { struct AADereferenceableImpl : AADereferenceable { AADereferenceableImpl(const IRPosition &IRP, Attributor &A) : AADereferenceable(IRP, A) {} @@ -4152,6 +4260,7 @@ struct AADereferenceableImpl : AADereferenceable { /// See AbstractAttribute::initialize(...). void initialize(Attributor &A) override { + Value &V = *getAssociatedValue().stripPointerCasts(); SmallVector<Attribute, 4> Attrs; getAttrs({Attribute::Dereferenceable, Attribute::DereferenceableOrNull}, Attrs, /* IgnoreSubsumingPositions */ false, &A); @@ -4162,9 +4271,8 @@ struct AADereferenceableImpl : AADereferenceable { NonNullAA = &A.getAAFor<AANonNull>(*this, IRP, DepClassTy::NONE); bool CanBeNull, CanBeFreed; - takeKnownDerefBytesMaximum( - IRP.getAssociatedValue().getPointerDereferenceableBytes( - A.getDataLayout(), CanBeNull, CanBeFreed)); + takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes( + A.getDataLayout(), CanBeNull, CanBeFreed)); bool IsFnInterface = IRP.isFnInterfaceKind(); Function *FnScope = IRP.getAnchorScope(); @@ -4263,8 +4371,9 @@ struct AADereferenceableFloating : AADereferenceableImpl { unsigned IdxWidth = DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace()); APInt Offset(IdxWidth, 0); - const Value *Base = - stripAndAccumulateMinimalOffsets(A, *this, &V, DL, Offset, false); + const Value *Base = stripAndAccumulateOffsets( + A, *this, &V, DL, Offset, /* GetMinOffset */ false, + /* AllowNonInbounds */ true); const auto &AA = A.getAAFor<AADereferenceable>( *this, IRPosition::value(*Base), DepClassTy::REQUIRED); @@ -4312,8 +4421,10 @@ struct AADereferenceableFloating : AADereferenceableImpl { }; DerefState T; + bool UsedAssumedInformation = false; if (!genericValueTraversal<DerefState>(A, getIRPosition(), *this, T, - VisitValueCB, getCtxI())) + VisitValueCB, getCtxI(), + UsedAssumedInformation)) return indicatePessimisticFixpoint(); return clampStateAndIndicateChange(getState(), T); @@ -4377,9 +4488,11 @@ struct AADereferenceableCallSiteReturned final STATS_DECLTRACK_CS_ATTR(dereferenceable); } }; +} // namespace // ------------------------ Align Argument Attribute ------------------------ +namespace { static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA, Value &AssociatedValue, const Use *U, const Instruction *I, bool &TrackUse) { @@ -4450,14 +4563,8 @@ struct AAAlignImpl : AAAlign { for (const Attribute &Attr : Attrs) takeKnownMaximum(Attr.getValueAsInt()); - Value &V = getAssociatedValue(); - // TODO: This is a HACK to avoid getPointerAlignment to introduce a ptr2int - // use of the function pointer. This was caused by D73131. We want to - // avoid this for function pointers especially because we iterate - // their uses and int2ptr is not handled. It is not a correctness - // problem though! - if (!V.getType()->getPointerElementType()->isFunctionTy()) - takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value()); + Value &V = *getAssociatedValue().stripPointerCasts(); + takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value()); if (getIRPosition().isFnInterfaceKind() && (!getAnchorScope() || @@ -4479,16 +4586,16 @@ struct AAAlignImpl : AAAlign { for (const Use &U : AssociatedValue.uses()) { if (auto *SI = dyn_cast<StoreInst>(U.getUser())) { if (SI->getPointerOperand() == &AssociatedValue) - if (SI->getAlignment() < getAssumedAlign()) { + if (SI->getAlign() < getAssumedAlign()) { STATS_DECLTRACK(AAAlign, Store, "Number of times alignment added to a store"); - SI->setAlignment(Align(getAssumedAlign())); + SI->setAlignment(getAssumedAlign()); LoadStoreChanged = ChangeStatus::CHANGED; } } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) { if (LI->getPointerOperand() == &AssociatedValue) - if (LI->getAlignment() < getAssumedAlign()) { - LI->setAlignment(Align(getAssumedAlign())); + if (LI->getAlign() < getAssumedAlign()) { + LI->setAlignment(getAssumedAlign()); STATS_DECLTRACK(AAAlign, Load, "Number of times alignment added to a load"); LoadStoreChanged = ChangeStatus::CHANGED; @@ -4532,9 +4639,8 @@ struct AAAlignImpl : AAAlign { /// See AbstractAttribute::getAsStr(). const std::string getAsStr() const override { - return getAssumedAlign() ? ("align<" + std::to_string(getKnownAlign()) + - "-" + std::to_string(getAssumedAlign()) + ">") - : "unknown-align"; + return "align<" + std::to_string(getKnownAlign().value()) + "-" + + std::to_string(getAssumedAlign().value()) + ">"; } }; @@ -4548,6 +4654,8 @@ struct AAAlignFloating : AAAlignImpl { auto VisitValueCB = [&](Value &V, const Instruction *, AAAlign::StateType &T, bool Stripped) -> bool { + if (isa<UndefValue>(V) || isa<ConstantPointerNull>(V)) + return true; const auto &AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V), DepClassTy::REQUIRED); if (!Stripped && this == &AA) { @@ -4555,6 +4663,7 @@ struct AAAlignFloating : AAAlignImpl { unsigned Alignment = 1; if (const Value *Base = GetPointerBaseWithConstantOffset(&V, Offset, DL)) { + // TODO: Use AAAlign for the base too. Align PA = Base->getPointerAlignment(DL); // BasePointerAddr + Offset = Alignment * Q for some integer Q. // So we can say that the maximum power of two which is a divisor of @@ -4578,8 +4687,10 @@ struct AAAlignFloating : AAAlignImpl { }; StateType T; + bool UsedAssumedInformation = false; if (!genericValueTraversal<StateType>(A, getIRPosition(), *this, T, - VisitValueCB, getCtxI())) + VisitValueCB, getCtxI(), + UsedAssumedInformation)) return indicatePessimisticFixpoint(); // TODO: If we know we visited all incoming values, thus no are assumed @@ -4657,7 +4768,7 @@ struct AAAlignCallSiteArgument final : AAAlignFloating { // so we do not need to track a dependence. const auto &ArgAlignAA = A.getAAFor<AAAlign>( *this, IRPosition::argument(*Arg), DepClassTy::NONE); - takeKnownMaximum(ArgAlignAA.getKnownAlign()); + takeKnownMaximum(ArgAlignAA.getKnownAlign().value()); } return Changed; } @@ -4684,8 +4795,10 @@ struct AAAlignCallSiteReturned final /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); } }; +} // namespace /// ------------------ Function No-Return Attribute ---------------------------- +namespace { struct AANoReturnImpl : public AANoReturn { AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {} @@ -4753,9 +4866,179 @@ struct AANoReturnCallSite final : AANoReturnImpl { /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); } }; +} // namespace + +/// ----------------------- Instance Info --------------------------------- + +namespace { +/// A class to hold the state of for no-capture attributes. +struct AAInstanceInfoImpl : public AAInstanceInfo { + AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A) + : AAInstanceInfo(IRP, A) {} + + /// See AbstractAttribute::initialize(...). + void initialize(Attributor &A) override { + Value &V = getAssociatedValue(); + if (auto *C = dyn_cast<Constant>(&V)) { + if (C->isThreadDependent()) + indicatePessimisticFixpoint(); + else + indicateOptimisticFixpoint(); + return; + } + if (auto *CB = dyn_cast<CallBase>(&V)) + if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() && + !CB->mayReadFromMemory()) { + indicateOptimisticFixpoint(); + return; + } + } + + /// See AbstractAttribute::updateImpl(...). + ChangeStatus updateImpl(Attributor &A) override { + ChangeStatus Changed = ChangeStatus::UNCHANGED; + + Value &V = getAssociatedValue(); + const Function *Scope = nullptr; + if (auto *I = dyn_cast<Instruction>(&V)) + Scope = I->getFunction(); + if (auto *A = dyn_cast<Argument>(&V)) { + Scope = A->getParent(); + if (!Scope->hasLocalLinkage()) + return Changed; + } + if (!Scope) + return indicateOptimisticFixpoint(); + + auto &NoRecurseAA = A.getAAFor<AANoRecurse>( + *this, IRPosition::function(*Scope), DepClassTy::OPTIONAL); + if (NoRecurseAA.isAssumedNoRecurse()) + return Changed; + + auto UsePred = [&](const Use &U, bool &Follow) { + const Instruction *UserI = dyn_cast<Instruction>(U.getUser()); + if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) || + isa<PHINode>(UserI) || isa<SelectInst>(UserI)) { + Follow = true; + return true; + } + if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) || + (isa<StoreInst>(UserI) && + cast<StoreInst>(UserI)->getValueOperand() != U.get())) + return true; + if (auto *CB = dyn_cast<CallBase>(UserI)) { + // This check is not guaranteeing uniqueness but for now that we cannot + // end up with two versions of \p U thinking it was one. + if (!CB->getCalledFunction() || + !CB->getCalledFunction()->hasLocalLinkage()) + return true; + if (!CB->isArgOperand(&U)) + return false; + const auto &ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>( + *this, IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U)), + DepClassTy::OPTIONAL); + if (!ArgInstanceInfoAA.isAssumedUniqueForAnalysis()) + return false; + // If this call base might reach the scope again we might forward the + // argument back here. This is very conservative. + if (AA::isPotentiallyReachable(A, *CB, *Scope, *this, nullptr)) + return false; + return true; + } + return false; + }; + + auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) { + if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) { + auto *Ptr = SI->getPointerOperand()->stripPointerCasts(); + if (isa<AllocaInst>(Ptr) && AA::isDynamicallyUnique(A, *this, *Ptr)) + return true; + auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction( + *SI->getFunction()); + if (isAllocationFn(Ptr, TLI) && AA::isDynamicallyUnique(A, *this, *Ptr)) + return true; + } + return false; + }; + + if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true, + DepClassTy::OPTIONAL, + /* IgnoreDroppableUses */ true, EquivalentUseCB)) + return indicatePessimisticFixpoint(); + + return Changed; + } + + /// See AbstractState::getAsStr(). + const std::string getAsStr() const override { + return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>"; + } + + /// See AbstractAttribute::trackStatistics() + void trackStatistics() const override {} +}; + +/// InstanceInfo attribute for floating values. +struct AAInstanceInfoFloating : AAInstanceInfoImpl { + AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A) + : AAInstanceInfoImpl(IRP, A) {} +}; + +/// NoCapture attribute for function arguments. +struct AAInstanceInfoArgument final : AAInstanceInfoFloating { + AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A) + : AAInstanceInfoFloating(IRP, A) {} +}; + +/// InstanceInfo attribute for call site arguments. +struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl { + AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A) + : AAInstanceInfoImpl(IRP, A) {} + + /// See AbstractAttribute::updateImpl(...). + ChangeStatus updateImpl(Attributor &A) override { + // TODO: Once we have call site specific value information we can provide + // call site specific liveness information and then it makes + // sense to specialize attributes for call sites arguments instead of + // redirecting requests to the callee argument. + Argument *Arg = getAssociatedArgument(); + if (!Arg) + return indicatePessimisticFixpoint(); + const IRPosition &ArgPos = IRPosition::argument(*Arg); + auto &ArgAA = + A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED); + return clampStateAndIndicateChange(getState(), ArgAA.getState()); + } +}; + +/// InstanceInfo attribute for function return value. +struct AAInstanceInfoReturned final : AAInstanceInfoImpl { + AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A) + : AAInstanceInfoImpl(IRP, A) { + llvm_unreachable("InstanceInfo is not applicable to function returns!"); + } + + /// See AbstractAttribute::initialize(...). + void initialize(Attributor &A) override { + llvm_unreachable("InstanceInfo is not applicable to function returns!"); + } + + /// See AbstractAttribute::updateImpl(...). + ChangeStatus updateImpl(Attributor &A) override { + llvm_unreachable("InstanceInfo is not applicable to function returns!"); + } +}; + +/// InstanceInfo attribute deduction for a call site return value. +struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating { + AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A) + : AAInstanceInfoFloating(IRP, A) {} +}; +} // namespace /// ----------------------- Variable Capturing --------------------------------- +namespace { /// A class to hold the state of for no-capture attributes. struct AANoCaptureImpl : public AANoCapture { AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {} @@ -4863,143 +5146,69 @@ struct AANoCaptureImpl : public AANoCapture { return "assumed not-captured-maybe-returned"; return "assumed-captured"; } -}; - -/// Attributor-aware capture tracker. -struct AACaptureUseTracker final : public CaptureTracker { - - /// Create a capture tracker that can lookup in-flight abstract attributes - /// through the Attributor \p A. - /// - /// If a use leads to a potential capture, \p CapturedInMemory is set and the - /// search is stopped. If a use leads to a return instruction, - /// \p CommunicatedBack is set to true and \p CapturedInMemory is not changed. - /// If a use leads to a ptr2int which may capture the value, - /// \p CapturedInInteger is set. If a use is found that is currently assumed - /// "no-capture-maybe-returned", the user is added to the \p PotentialCopies - /// set. All values in \p PotentialCopies are later tracked as well. For every - /// explored use we decrement \p RemainingUsesToExplore. Once it reaches 0, - /// the search is stopped with \p CapturedInMemory and \p CapturedInInteger - /// conservatively set to true. - AACaptureUseTracker(Attributor &A, AANoCapture &NoCaptureAA, - const AAIsDead &IsDeadAA, AANoCapture::StateType &State, - SmallSetVector<Value *, 4> &PotentialCopies, - unsigned &RemainingUsesToExplore) - : A(A), NoCaptureAA(NoCaptureAA), IsDeadAA(IsDeadAA), State(State), - PotentialCopies(PotentialCopies), - RemainingUsesToExplore(RemainingUsesToExplore) {} - - /// Determine if \p V maybe captured. *Also updates the state!* - bool valueMayBeCaptured(const Value *V) { - if (V->getType()->isPointerTy()) { - PointerMayBeCaptured(V, this); - } else { - State.indicatePessimisticFixpoint(); - } - return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED); - } - - /// See CaptureTracker::tooManyUses(). - void tooManyUses() override { - State.removeAssumedBits(AANoCapture::NO_CAPTURE); - } - bool isDereferenceableOrNull(Value *O, const DataLayout &DL) override { - if (CaptureTracker::isDereferenceableOrNull(O, DL)) - return true; - const auto &DerefAA = A.getAAFor<AADereferenceable>( - NoCaptureAA, IRPosition::value(*O), DepClassTy::OPTIONAL); - return DerefAA.getAssumedDereferenceableBytes(); - } - - /// See CaptureTracker::captured(...). - bool captured(const Use *U) override { - Instruction *UInst = cast<Instruction>(U->getUser()); - LLVM_DEBUG(dbgs() << "Check use: " << *U->get() << " in " << *UInst - << "\n"); - - // Because we may reuse the tracker multiple times we keep track of the - // number of explored uses ourselves as well. - if (RemainingUsesToExplore-- == 0) { - LLVM_DEBUG(dbgs() << " - too many uses to explore!\n"); - return isCapturedIn(/* Memory */ true, /* Integer */ true, - /* Return */ true); - } + /// Check the use \p U and update \p State accordingly. Return true if we + /// should continue to update the state. + bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U, + bool &Follow) { + Instruction *UInst = cast<Instruction>(U.getUser()); + LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in " + << *UInst << "\n"); // Deal with ptr2int by following uses. if (isa<PtrToIntInst>(UInst)) { LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n"); - return valueMayBeCaptured(UInst); + return isCapturedIn(State, /* Memory */ true, /* Integer */ true, + /* Return */ true); } - // For stores we check if we can follow the value through memory or not. - if (auto *SI = dyn_cast<StoreInst>(UInst)) { - if (SI->isVolatile()) - return isCapturedIn(/* Memory */ true, /* Integer */ false, - /* Return */ false); - bool UsedAssumedInformation = false; - if (!AA::getPotentialCopiesOfStoredValue( - A, *SI, PotentialCopies, NoCaptureAA, UsedAssumedInformation)) - return isCapturedIn(/* Memory */ true, /* Integer */ false, - /* Return */ false); - // Not captured directly, potential copies will be checked. - return isCapturedIn(/* Memory */ false, /* Integer */ false, + // For stores we already checked if we can follow them, if they make it + // here we give up. + if (isa<StoreInst>(UInst)) + return isCapturedIn(State, /* Memory */ true, /* Integer */ false, /* Return */ false); - } // Explicitly catch return instructions. if (isa<ReturnInst>(UInst)) { - if (UInst->getFunction() == NoCaptureAA.getAnchorScope()) - return isCapturedIn(/* Memory */ false, /* Integer */ false, + if (UInst->getFunction() == getAnchorScope()) + return isCapturedIn(State, /* Memory */ false, /* Integer */ false, /* Return */ true); - return isCapturedIn(/* Memory */ true, /* Integer */ true, + return isCapturedIn(State, /* Memory */ true, /* Integer */ true, /* Return */ true); } // For now we only use special logic for call sites. However, the tracker // itself knows about a lot of other non-capturing cases already. auto *CB = dyn_cast<CallBase>(UInst); - if (!CB || !CB->isArgOperand(U)) - return isCapturedIn(/* Memory */ true, /* Integer */ true, + if (!CB || !CB->isArgOperand(&U)) + return isCapturedIn(State, /* Memory */ true, /* Integer */ true, /* Return */ true); - unsigned ArgNo = CB->getArgOperandNo(U); + unsigned ArgNo = CB->getArgOperandNo(&U); const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo); // If we have a abstract no-capture attribute for the argument we can use // it to justify a non-capture attribute here. This allows recursion! auto &ArgNoCaptureAA = - A.getAAFor<AANoCapture>(NoCaptureAA, CSArgPos, DepClassTy::REQUIRED); + A.getAAFor<AANoCapture>(*this, CSArgPos, DepClassTy::REQUIRED); if (ArgNoCaptureAA.isAssumedNoCapture()) - return isCapturedIn(/* Memory */ false, /* Integer */ false, + return isCapturedIn(State, /* Memory */ false, /* Integer */ false, /* Return */ false); if (ArgNoCaptureAA.isAssumedNoCaptureMaybeReturned()) { - addPotentialCopy(*CB); - return isCapturedIn(/* Memory */ false, /* Integer */ false, + Follow = true; + return isCapturedIn(State, /* Memory */ false, /* Integer */ false, /* Return */ false); } // Lastly, we could not find a reason no-capture can be assumed so we don't. - return isCapturedIn(/* Memory */ true, /* Integer */ true, + return isCapturedIn(State, /* Memory */ true, /* Integer */ true, /* Return */ true); } - /// Register \p CS as potential copy of the value we are checking. - void addPotentialCopy(CallBase &CB) { PotentialCopies.insert(&CB); } - - /// See CaptureTracker::shouldExplore(...). - bool shouldExplore(const Use *U) override { - // Check liveness and ignore droppable users. - bool UsedAssumedInformation = false; - return !U->getUser()->isDroppable() && - !A.isAssumedDead(*U, &NoCaptureAA, &IsDeadAA, - UsedAssumedInformation); - } - - /// Update the state according to \p CapturedInMem, \p CapturedInInt, and - /// \p CapturedInRet, then return the appropriate value for use in the - /// CaptureTracker::captured() interface. - bool isCapturedIn(bool CapturedInMem, bool CapturedInInt, - bool CapturedInRet) { + /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and + /// \p CapturedInRet, then return true if we should continue updating the + /// state. + static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem, + bool CapturedInInt, bool CapturedInRet) { LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int " << CapturedInInt << "|Ret " << CapturedInRet << "]\n"); if (CapturedInMem) @@ -5008,27 +5217,8 @@ struct AACaptureUseTracker final : public CaptureTracker { State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT); if (CapturedInRet) State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET); - return !State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED); + return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED); } - -private: - /// The attributor providing in-flight abstract attributes. - Attributor &A; - - /// The abstract attribute currently updated. - AANoCapture &NoCaptureAA; - - /// The abstract liveness state. - const AAIsDead &IsDeadAA; - - /// The state currently updated. - AANoCapture::StateType &State; - - /// Set of potential copies of the tracked value. - SmallSetVector<Value *, 4> &PotentialCopies; - - /// Global counter to limit the number of explored uses. - unsigned &RemainingUsesToExplore; }; ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) { @@ -5042,7 +5232,6 @@ ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) { isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope(); assert(F && "Expected a function!"); const IRPosition &FnPos = IRPosition::function(*F); - const auto &IsDeadAA = A.getAAFor<AAIsDead>(*this, FnPos, DepClassTy::NONE); AANoCapture::StateType T; @@ -5059,6 +5248,8 @@ ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) { // AAReturnedValues, e.g., track all values that escape through returns // directly somehow. auto CheckReturnedArgs = [&](const AAReturnedValues &RVAA) { + if (!RVAA.getState().isValidState()) + return false; bool SeenConstant = false; for (auto &It : RVAA.returned_values()) { if (isa<Constant>(It.first)) { @@ -5094,21 +5285,27 @@ ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) { } } - // Use the CaptureTracker interface and logic with the specialized tracker, - // defined in AACaptureUseTracker, that can look at in-flight abstract - // attributes and directly updates the assumed state. - SmallSetVector<Value *, 4> PotentialCopies; - unsigned RemainingUsesToExplore = - getDefaultMaxUsesToExploreForCaptureTracking(); - AACaptureUseTracker Tracker(A, *this, IsDeadAA, T, PotentialCopies, - RemainingUsesToExplore); + auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) { + const auto &DerefAA = A.getAAFor<AADereferenceable>( + *this, IRPosition::value(*O), DepClassTy::OPTIONAL); + return DerefAA.getAssumedDereferenceableBytes(); + }; - // Check all potential copies of the associated value until we can assume - // none will be captured or we have to assume at least one might be. - unsigned Idx = 0; - PotentialCopies.insert(V); - while (T.isAssumed(NO_CAPTURE_MAYBE_RETURNED) && Idx < PotentialCopies.size()) - Tracker.valueMayBeCaptured(PotentialCopies[Idx++]); + auto UseCheck = [&](const Use &U, bool &Follow) -> bool { + switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) { + case UseCaptureKind::NO_CAPTURE: + return true; + case UseCaptureKind::MAY_CAPTURE: + return checkUse(A, T, U, Follow); + case UseCaptureKind::PASSTHROUGH: + Follow = true; + return true; + } + llvm_unreachable("Unexpected use capture kind!"); + }; + + if (!A.checkForAllUses(UseCheck, *this, *V)) + return indicatePessimisticFixpoint(); AANoCapture::StateType &S = getState(); auto Assumed = S.getAssumed(); @@ -5208,6 +5405,7 @@ struct AANoCaptureCallSiteReturned final : AANoCaptureImpl { STATS_DECLTRACK_CSRET_ATTR(nocapture) } }; +} // namespace /// ------------------ Value Simplify Attribute ---------------------------- @@ -5219,7 +5417,7 @@ bool ValueSimplifyStateType::unionAssumed(Optional<Value *> Other) { return false; LLVM_DEBUG({ - if (SimplifiedAssociatedValue.hasValue()) + if (SimplifiedAssociatedValue) dbgs() << "[ValueSimplify] is assumed to be " << **SimplifiedAssociatedValue << "\n"; else @@ -5228,6 +5426,7 @@ bool ValueSimplifyStateType::unionAssumed(Optional<Value *> Other) { return true; } +namespace { struct AAValueSimplifyImpl : AAValueSimplify { AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A) : AAValueSimplify(IRP, A) {} @@ -5243,9 +5442,9 @@ struct AAValueSimplifyImpl : AAValueSimplify { /// See AbstractAttribute::getAsStr(). const std::string getAsStr() const override { LLVM_DEBUG({ - errs() << "SAV: " << SimplifiedAssociatedValue << " "; + dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " "; if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue) - errs() << "SAV: " << **SimplifiedAssociatedValue << " "; + dbgs() << "SAV: " << **SimplifiedAssociatedValue << " "; }); return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple") : "not-simple"; @@ -5259,24 +5458,101 @@ struct AAValueSimplifyImpl : AAValueSimplify { return SimplifiedAssociatedValue; } + /// Ensure the return value is \p V with type \p Ty, if not possible return + /// nullptr. If \p Check is true we will only verify such an operation would + /// suceed and return a non-nullptr value if that is the case. No IR is + /// generated or modified. + static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI, + bool Check) { + if (auto *TypedV = AA::getWithType(V, Ty)) + return TypedV; + if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty)) + return Check ? &V + : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(&V, &Ty, + "", CtxI); + return nullptr; + } + + /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble. + /// If \p Check is true we will only verify such an operation would suceed and + /// return a non-nullptr value if that is the case. No IR is generated or + /// modified. + static Value *reproduceInst(Attributor &A, + const AbstractAttribute &QueryingAA, + Instruction &I, Type &Ty, Instruction *CtxI, + bool Check, ValueToValueMapTy &VMap) { + assert(CtxI && "Cannot reproduce an instruction without context!"); + if (Check && (I.mayReadFromMemory() || + !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr, + /* TLI */ nullptr))) + return nullptr; + for (Value *Op : I.operands()) { + Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap); + if (!NewOp) { + assert(Check && "Manifest of new value unexpectedly failed!"); + return nullptr; + } + if (!Check) + VMap[Op] = NewOp; + } + if (Check) + return &I; + + Instruction *CloneI = I.clone(); + // TODO: Try to salvage debug information here. + CloneI->setDebugLoc(DebugLoc()); + VMap[&I] = CloneI; + CloneI->insertBefore(CtxI); + RemapInstruction(CloneI, VMap); + return CloneI; + } + + /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble. + /// If \p Check is true we will only verify such an operation would suceed and + /// return a non-nullptr value if that is the case. No IR is generated or + /// modified. + static Value *reproduceValue(Attributor &A, + const AbstractAttribute &QueryingAA, Value &V, + Type &Ty, Instruction *CtxI, bool Check, + ValueToValueMapTy &VMap) { + if (const auto &NewV = VMap.lookup(&V)) + return NewV; + bool UsedAssumedInformation = false; + Optional<Value *> SimpleV = + A.getAssumedSimplified(V, QueryingAA, UsedAssumedInformation); + if (!SimpleV) + return PoisonValue::get(&Ty); + Value *EffectiveV = &V; + if (SimpleV.getValue()) + EffectiveV = SimpleV.getValue(); + if (auto *C = dyn_cast<Constant>(EffectiveV)) + if (!C->canTrap()) + return C; + if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI), + A.getInfoCache())) + return ensureType(A, *EffectiveV, Ty, CtxI, Check); + if (auto *I = dyn_cast<Instruction>(EffectiveV)) + if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap)) + return ensureType(A, *NewV, Ty, CtxI, Check); + return nullptr; + } + /// Return a value we can use as replacement for the associated one, or /// nullptr if we don't have one that makes sense. - Value *getReplacementValue(Attributor &A) const { - Value *NewV; - NewV = SimplifiedAssociatedValue.hasValue() - ? SimplifiedAssociatedValue.getValue() - : UndefValue::get(getAssociatedType()); - if (!NewV) - return nullptr; - NewV = AA::getWithType(*NewV, *getAssociatedType()); - if (!NewV || NewV == &getAssociatedValue()) - return nullptr; - const Instruction *CtxI = getCtxI(); - if (CtxI && !AA::isValidAtPosition(*NewV, *CtxI, A.getInfoCache())) - return nullptr; - if (!CtxI && !AA::isValidInScope(*NewV, getAnchorScope())) - return nullptr; - return NewV; + Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const { + Value *NewV = SimplifiedAssociatedValue + ? SimplifiedAssociatedValue.getValue() + : UndefValue::get(getAssociatedType()); + if (NewV && NewV != &getAssociatedValue()) { + ValueToValueMapTy VMap; + // First verify we can reprduce the value with the required type at the + // context location before we actually start modifying the IR. + if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI, + /* CheckOnly */ true, VMap)) + return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI, + /* CheckOnly */ false, VMap); + } + return nullptr; } /// Helper function for querying AAValueSimplify and updating candicate. @@ -5300,14 +5576,14 @@ struct AAValueSimplifyImpl : AAValueSimplify { const auto &AA = A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE); - Optional<ConstantInt *> COpt = AA.getAssumedConstantInt(A); + Optional<Constant *> COpt = AA.getAssumedConstant(A); - if (!COpt.hasValue()) { + if (!COpt) { SimplifiedAssociatedValue = llvm::None; A.recordDependence(AA, *this, DepClassTy::OPTIONAL); return true; } - if (auto *C = COpt.getValue()) { + if (auto *C = *COpt) { SimplifiedAssociatedValue = C; A.recordDependence(AA, *this, DepClassTy::OPTIONAL); return true; @@ -5318,7 +5594,7 @@ struct AAValueSimplifyImpl : AAValueSimplify { bool askSimplifiedValueForOtherAAs(Attributor &A) { if (askSimplifiedValueFor<AAValueConstantRange>(A)) return true; - if (askSimplifiedValueFor<AAPotentialValues>(A)) + if (askSimplifiedValueFor<AAPotentialConstantValues>(A)) return true; return false; } @@ -5326,14 +5602,18 @@ struct AAValueSimplifyImpl : AAValueSimplify { /// See AbstractAttribute::manifest(...). ChangeStatus manifest(Attributor &A) override { ChangeStatus Changed = ChangeStatus::UNCHANGED; - if (getAssociatedValue().user_empty()) - return Changed; - - if (auto *NewV = getReplacementValue(A)) { - LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue() << " -> " - << *NewV << " :: " << *this << "\n"); - if (A.changeValueAfterManifest(getAssociatedValue(), *NewV)) - Changed = ChangeStatus::CHANGED; + for (auto &U : getAssociatedValue().uses()) { + // Check if we need to adjust the insertion point to make sure the IR is + // valid. + Instruction *IP = dyn_cast<Instruction>(U.getUser()); + if (auto *PHI = dyn_cast_or_null<PHINode>(IP)) + IP = PHI->getIncomingBlock(U)->getTerminator(); + if (auto *NewV = manifestReplacementValue(A, IP)) { + LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue() + << " -> " << *NewV << " :: " << *this << "\n"); + if (A.changeUseAfterManifest(U, *NewV)) + Changed = ChangeStatus::CHANGED; + } } return Changed | AAValueSimplify::manifest(A); @@ -5344,73 +5624,6 @@ struct AAValueSimplifyImpl : AAValueSimplify { SimplifiedAssociatedValue = &getAssociatedValue(); return AAValueSimplify::indicatePessimisticFixpoint(); } - - static bool handleLoad(Attributor &A, const AbstractAttribute &AA, - LoadInst &L, function_ref<bool(Value &)> Union) { - auto UnionWrapper = [&](Value &V, Value &Obj) { - if (isa<AllocaInst>(Obj)) - return Union(V); - if (!AA::isDynamicallyUnique(A, AA, V)) - return false; - if (!AA::isValidAtPosition(V, L, A.getInfoCache())) - return false; - return Union(V); - }; - - Value &Ptr = *L.getPointerOperand(); - SmallVector<Value *, 8> Objects; - if (!AA::getAssumedUnderlyingObjects(A, Ptr, Objects, AA, &L)) - return false; - - const auto *TLI = - A.getInfoCache().getTargetLibraryInfoForFunction(*L.getFunction()); - for (Value *Obj : Objects) { - LLVM_DEBUG(dbgs() << "Visit underlying object " << *Obj << "\n"); - if (isa<UndefValue>(Obj)) - continue; - if (isa<ConstantPointerNull>(Obj)) { - // A null pointer access can be undefined but any offset from null may - // be OK. We do not try to optimize the latter. - bool UsedAssumedInformation = false; - if (!NullPointerIsDefined(L.getFunction(), - Ptr.getType()->getPointerAddressSpace()) && - A.getAssumedSimplified(Ptr, AA, UsedAssumedInformation) == Obj) - continue; - return false; - } - Constant *InitialVal = AA::getInitialValueForObj(*Obj, *L.getType(), TLI); - if (!InitialVal || !Union(*InitialVal)) - return false; - - LLVM_DEBUG(dbgs() << "Underlying object amenable to load-store " - "propagation, checking accesses next.\n"); - - auto CheckAccess = [&](const AAPointerInfo::Access &Acc, bool IsExact) { - LLVM_DEBUG(dbgs() << " - visit access " << Acc << "\n"); - if (Acc.isWrittenValueYetUndetermined()) - return true; - Value *Content = Acc.getWrittenValue(); - if (!Content) - return false; - Value *CastedContent = - AA::getWithType(*Content, *AA.getAssociatedType()); - if (!CastedContent) - return false; - if (IsExact) - return UnionWrapper(*CastedContent, *Obj); - if (auto *C = dyn_cast<Constant>(CastedContent)) - if (C->isNullValue() || C->isAllOnesValue() || isa<UndefValue>(C)) - return UnionWrapper(*CastedContent, *Obj); - return false; - }; - - auto &PI = A.getAAFor<AAPointerInfo>(AA, IRPosition::value(*Obj), - DepClassTy::REQUIRED); - if (!PI.forallInterferingWrites(A, AA, L, CheckAccess)) - return false; - } - return true; - } }; struct AAValueSimplifyArgument final : AAValueSimplifyImpl { @@ -5425,15 +5638,6 @@ struct AAValueSimplifyArgument final : AAValueSimplifyImpl { Attribute::StructRet, Attribute::Nest, Attribute::ByVal}, /* IgnoreSubsumingPositions */ true)) indicatePessimisticFixpoint(); - - // FIXME: This is a hack to prevent us from propagating function poiner in - // the new pass manager CGSCC pass as it creates call edges the - // CallGraphUpdater cannot handle yet. - Value &V = getAssociatedValue(); - if (V.getType()->isPointerTy() && - V.getType()->getPointerElementType()->isFunctionTy() && - !A.isModulePass()) - indicatePessimisticFixpoint(); } /// See AbstractAttribute::updateImpl(...). @@ -5466,7 +5670,7 @@ struct AAValueSimplifyArgument final : AAValueSimplifyImpl { bool UsedAssumedInformation = false; Optional<Constant *> SimpleArgOp = A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation); - if (!SimpleArgOp.hasValue()) + if (!SimpleArgOp) return true; if (!SimpleArgOp.getValue()) return false; @@ -5477,14 +5681,14 @@ struct AAValueSimplifyArgument final : AAValueSimplifyImpl { // Generate a answer specific to a call site context. bool Success; - bool AllCallSitesKnown; + bool UsedAssumedInformation = false; if (hasCallBaseContext() && getCallBaseContext()->getCalledFunction() == Arg->getParent()) Success = PredForCallSite( AbstractCallSite(&getCallBaseContext()->getCalledOperandUse())); else Success = A.checkForAllCallSites(PredForCallSite, *this, true, - AllCallSitesKnown); + UsedAssumedInformation); if (!Success) if (!askSimplifiedValueForOtherAAs(A)) @@ -5516,12 +5720,16 @@ struct AAValueSimplifyReturned : AAValueSimplifyImpl { ChangeStatus updateImpl(Attributor &A) override { auto Before = SimplifiedAssociatedValue; - auto PredForReturned = [&](Value &V) { - return checkAndUpdate(A, *this, - IRPosition::value(V, getCallBaseContext())); + auto ReturnInstCB = [&](Instruction &I) { + auto &RI = cast<ReturnInst>(I); + return checkAndUpdate( + A, *this, + IRPosition::value(*RI.getReturnValue(), getCallBaseContext())); }; - if (!A.checkForAllReturnedValues(PredForReturned, *this)) + bool UsedAssumedInformation = false; + if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret}, + UsedAssumedInformation)) if (!askSimplifiedValueForOtherAAs(A)) return indicatePessimisticFixpoint(); @@ -5531,29 +5739,9 @@ struct AAValueSimplifyReturned : AAValueSimplifyImpl { } ChangeStatus manifest(Attributor &A) override { - ChangeStatus Changed = ChangeStatus::UNCHANGED; - - if (auto *NewV = getReplacementValue(A)) { - auto PredForReturned = - [&](Value &, const SmallSetVector<ReturnInst *, 4> &RetInsts) { - for (ReturnInst *RI : RetInsts) { - Value *ReturnedVal = RI->getReturnValue(); - if (ReturnedVal == NewV || isa<UndefValue>(ReturnedVal)) - return true; - assert(RI->getFunction() == getAnchorScope() && - "ReturnInst in wrong function!"); - LLVM_DEBUG(dbgs() - << "[ValueSimplify] " << *ReturnedVal << " -> " - << *NewV << " in " << *RI << " :: " << *this << "\n"); - if (A.changeUseAfterManifest(RI->getOperandUse(0), *NewV)) - Changed = ChangeStatus::CHANGED; - } - return true; - }; - A.checkForAllReturnedValuesAndReturnInsts(PredForReturned, *this); - } - - return Changed | AAValueSimplify::manifest(A); + // We queried AAValueSimplify for the returned values so they will be + // replaced if a simplified form was found. Nothing to do here. + return ChangeStatus::UNCHANGED; } /// See AbstractAttribute::trackStatistics() @@ -5597,7 +5785,7 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { const auto &SimplifiedLHS = A.getAssumedSimplified(IRPosition::value(*LHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedLHS.hasValue()) + if (!SimplifiedLHS) return true; if (!SimplifiedLHS.getValue()) return false; @@ -5606,7 +5794,7 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { const auto &SimplifiedRHS = A.getAssumedSimplified(IRPosition::value(*RHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedRHS.hasValue()) + if (!SimplifiedRHS) return true; if (!SimplifiedRHS.getValue()) return false; @@ -5662,15 +5850,6 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { return true; } - bool updateWithLoad(Attributor &A, LoadInst &L) { - auto Union = [&](Value &V) { - SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice( - SimplifiedAssociatedValue, &V, L.getType()); - return SimplifiedAssociatedValue != Optional<Value *>(nullptr); - }; - return handleLoad(A, *this, L, Union); - } - /// Use the generic, non-optimistic InstSimplfy functionality if we managed to /// simplify any operand of the instruction \p I. Return true if successful, /// in that case SimplifiedAssociatedValue will be updated. @@ -5686,7 +5865,7 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { *this, UsedAssumedInformation); // If we are not sure about any operand we are not sure about the entire // instruction, we'll wait. - if (!SimplifiedOp.hasValue()) + if (!SimplifiedOp) return true; if (SimplifiedOp.getValue()) @@ -5714,7 +5893,7 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { const DataLayout &DL = I.getModule()->getDataLayout(); SimplifyQuery Q(DL, TLI, DT, AC, &I); if (Value *SimplifiedI = - SimplifyInstructionWithOperands(&I, NewOps, Q, ORE)) { + simplifyInstructionWithOperands(&I, NewOps, Q, ORE)) { SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice( SimplifiedAssociatedValue, SimplifiedI, I.getType()); return SimplifiedAssociatedValue != Optional<Value *>(nullptr); @@ -5726,6 +5905,36 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { ChangeStatus updateImpl(Attributor &A) override { auto Before = SimplifiedAssociatedValue; + // Do not simplify loads that are only used in llvm.assume if we cannot also + // remove all stores that may feed into the load. The reason is that the + // assume is probably worth something as long as the stores are around. + if (auto *LI = dyn_cast<LoadInst>(&getAssociatedValue())) { + InformationCache &InfoCache = A.getInfoCache(); + if (InfoCache.isOnlyUsedByAssume(*LI)) { + SmallSetVector<Value *, 4> PotentialCopies; + SmallSetVector<Instruction *, 4> PotentialValueOrigins; + bool UsedAssumedInformation = false; + if (AA::getPotentiallyLoadedValues(A, *LI, PotentialCopies, + PotentialValueOrigins, *this, + UsedAssumedInformation, + /* OnlyExact */ true)) { + if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) { + if (!I) + return true; + if (auto *SI = dyn_cast<StoreInst>(I)) + return A.isAssumedDead(SI->getOperandUse(0), this, + /* LivenessAA */ nullptr, + UsedAssumedInformation, + /* CheckBBLivenessOnly */ false); + return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr, + UsedAssumedInformation, + /* CheckBBLivenessOnly */ false); + })) + return indicatePessimisticFixpoint(); + } + } + } + auto VisitValueCB = [&](Value &V, const Instruction *CtxI, bool &, bool Stripped) -> bool { auto &AA = A.getAAFor<AAValueSimplify>( @@ -5734,9 +5943,6 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { if (!Stripped && this == &AA) { if (auto *I = dyn_cast<Instruction>(&V)) { - if (auto *LI = dyn_cast<LoadInst>(&V)) - if (updateWithLoad(A, *LI)) - return true; if (auto *Cmp = dyn_cast<CmpInst>(&V)) if (handleCmp(A, *Cmp)) return true; @@ -5754,8 +5960,10 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl { }; bool Dummy = false; + bool UsedAssumedInformation = false; if (!genericValueTraversal<bool>(A, getIRPosition(), *this, Dummy, VisitValueCB, getCtxI(), + UsedAssumedInformation, /* UseValueSimplify */ false)) if (!askSimplifiedValueForOtherAAs(A)) return indicatePessimisticFixpoint(); @@ -5806,8 +6014,23 @@ struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl { void initialize(Attributor &A) override { AAValueSimplifyImpl::initialize(A); - if (!getAssociatedFunction()) + Function *Fn = getAssociatedFunction(); + if (!Fn) { indicatePessimisticFixpoint(); + return; + } + for (Argument &Arg : Fn->args()) { + if (Arg.hasReturnedAttr()) { + auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()), + Arg.getArgNo()); + if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT && + checkAndUpdate(A, *this, IRP)) + indicateOptimisticFixpoint(); + else + indicatePessimisticFixpoint(); + return; + } + } } /// See AbstractAttribute::updateImpl(...). @@ -5845,8 +6068,13 @@ struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating { /// See AbstractAttribute::manifest(...). ChangeStatus manifest(Attributor &A) override { ChangeStatus Changed = ChangeStatus::UNCHANGED; + // TODO: We should avoid simplification duplication to begin with. + auto *FloatAA = A.lookupAAFor<AAValueSimplify>( + IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE); + if (FloatAA && FloatAA->getState().isValidState()) + return Changed; - if (auto *NewV = getReplacementValue(A)) { + if (auto *NewV = manifestReplacementValue(A, getCtxI())) { Use &U = cast<CallBase>(&getAnchorValue()) ->getArgOperandUse(getCallSiteArgNo()); if (A.changeUseAfterManifest(U, *NewV)) @@ -5860,8 +6088,10 @@ struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating { STATS_DECLTRACK_CSARG_ATTR(value_simplify) } }; +} // namespace /// ----------------------- Heap-To-Stack Conversion --------------------------- +namespace { struct AAHeapToStackFunction final : public AAHeapToStack { struct AllocationInfo { @@ -5883,7 +6113,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack { bool HasPotentiallyFreeingUnknownUses = false; /// The set of free calls that use this allocation. - SmallPtrSet<CallBase *, 1> PotentialFreeCalls{}; + SmallSetVector<CallBase *, 1> PotentialFreeCalls{}; }; struct DeallocationInfo { @@ -5895,7 +6125,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack { bool MightFreeUnknownObjects = false; /// The set of allocation calls that are potentially freed. - SmallPtrSet<CallBase *, 1> PotentialAllocationCalls{}; + SmallSetVector<CallBase *, 1> PotentialAllocationCalls{}; }; AAHeapToStackFunction(const IRPosition &IRP, Attributor &A) @@ -5905,9 +6135,9 @@ struct AAHeapToStackFunction final : public AAHeapToStack { // Ensure we call the destructor so we release any memory allocated in the // sets. for (auto &It : AllocationInfos) - It.getSecond()->~AllocationInfo(); + It.second->~AllocationInfo(); for (auto &It : DeallocationInfos) - It.getSecond()->~DeallocationInfo(); + It.second->~DeallocationInfo(); } void initialize(Attributor &A) override { @@ -5932,7 +6162,8 @@ struct AAHeapToStackFunction final : public AAHeapToStack { if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) { AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB}; AllocationInfos[CB] = AI; - TLI->getLibFunc(*CB, AI->LibraryFunctionId); + if (TLI) + TLI->getLibFunc(*CB, AI->LibraryFunctionId); } } return true; @@ -5945,6 +6176,16 @@ struct AAHeapToStackFunction final : public AAHeapToStack { /* CheckPotentiallyDead */ true); (void)Success; assert(Success && "Did not expect the call base visit callback to fail!"); + + Attributor::SimplifictionCallbackTy SCB = + [](const IRPosition &, const AbstractAttribute *, + bool &) -> Optional<Value *> { return nullptr; }; + for (const auto &It : AllocationInfos) + A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first), + SCB); + for (const auto &It : DeallocationInfos) + A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first), + SCB); } const std::string getAsStr() const override { @@ -5971,7 +6212,8 @@ struct AAHeapToStackFunction final : public AAHeapToStack { bool isAssumedHeapToStack(const CallBase &CB) const override { if (isValidState()) - if (AllocationInfo *AI = AllocationInfos.lookup(&CB)) + if (AllocationInfo *AI = + AllocationInfos.lookup(const_cast<CallBase *>(&CB))) return AI->Status != AllocationInfo::INVALID; return false; } @@ -6000,6 +6242,17 @@ struct AAHeapToStackFunction final : public AAHeapToStack { Function *F = getAnchorScope(); const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F); + LoopInfo *LI = + A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F); + Optional<bool> MayContainIrreducibleControl; + auto IsInLoop = [&](BasicBlock &BB) { + if (!MayContainIrreducibleControl.has_value()) + MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI); + if (MayContainIrreducibleControl.value()) + return true; + return LI->getLoopFor(&BB) != nullptr; + }; + for (auto &It : AllocationInfos) { AllocationInfo &AI = *It.second; if (AI.Status == AllocationInfo::INVALID) @@ -6026,13 +6279,13 @@ struct AAHeapToStackFunction final : public AAHeapToStack { else A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark); + const DataLayout &DL = A.getInfoCache().getDL(); Value *Size; Optional<APInt> SizeAPI = getSize(A, *this, AI); - if (SizeAPI.hasValue()) { + if (SizeAPI) { Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI); } else { LLVMContext &Ctx = AI.CB->getContext(); - auto &DL = A.getInfoCache().getDL(); ObjectSizeOpts Opts; ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts); SizeOffsetEvalType SizeOffsetPair = Eval.compute(AI.CB); @@ -6041,32 +6294,36 @@ struct AAHeapToStackFunction final : public AAHeapToStack { Size = SizeOffsetPair.first; } + Instruction *IP = (!SizeAPI.has_value() || IsInLoop(*AI.CB->getParent())) + ? AI.CB + : &F->getEntryBlock().front(); + Align Alignment(1); if (MaybeAlign RetAlign = AI.CB->getRetAlign()) - Alignment = max(Alignment, RetAlign); + Alignment = std::max(Alignment, *RetAlign); if (Value *Align = getAllocAlignment(AI.CB, TLI)) { Optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align); - assert(AlignmentAPI.hasValue() && + assert(AlignmentAPI && AlignmentAPI.getValue().getZExtValue() > 0 && "Expected an alignment during manifest!"); - Alignment = - max(Alignment, MaybeAlign(AlignmentAPI.getValue().getZExtValue())); + Alignment = std::max( + Alignment, assumeAligned(AlignmentAPI.getValue().getZExtValue())); } - unsigned AS = cast<PointerType>(AI.CB->getType())->getAddressSpace(); - Instruction *Alloca = - new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment, - "", AI.CB->getNextNode()); + // TODO: Hoist the alloca towards the function entry. + unsigned AS = DL.getAllocaAddrSpace(); + Instruction *Alloca = new AllocaInst(Type::getInt8Ty(F->getContext()), AS, + Size, Alignment, "", IP); if (Alloca->getType() != AI.CB->getType()) - Alloca = new BitCastInst(Alloca, AI.CB->getType(), "malloc_bc", - Alloca->getNextNode()); + Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( + Alloca, AI.CB->getType(), "malloc_cast", AI.CB); auto *I8Ty = Type::getInt8Ty(F->getContext()); auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty); assert(InitVal && "Must be able to materialize initial memory state of allocation"); - A.changeValueAfterManifest(*AI.CB, *Alloca); + A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca); if (auto *II = dyn_cast<InvokeInst>(AI.CB)) { auto *NBB = II->getNormalDest(); @@ -6095,7 +6352,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack { bool UsedAssumedInformation = false; Optional<Constant *> SimpleV = A.getAssumedConstant(V, AA, UsedAssumedInformation); - if (!SimpleV.hasValue()) + if (!SimpleV) return APInt(64, 0); if (auto *CI = dyn_cast_or_null<ConstantInt>(SimpleV.getValue())) return CI->getValue(); @@ -6120,11 +6377,11 @@ struct AAHeapToStackFunction final : public AAHeapToStack { /// Collection of all malloc-like calls in a function with associated /// information. - DenseMap<CallBase *, AllocationInfo *> AllocationInfos; + MapVector<CallBase *, AllocationInfo *> AllocationInfos; /// Collection of all free-like calls in a function with associated /// information. - DenseMap<CallBase *, DeallocationInfo *> DeallocationInfos; + MapVector<CallBase *, DeallocationInfo *> DeallocationInfos; ChangeStatus updateImpl(Attributor &A) override; }; @@ -6167,7 +6424,8 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { // branches etc. SmallVector<Value *, 8> Objects; if (!AA::getAssumedUnderlyingObjects(A, *DI.CB->getArgOperand(0), Objects, - *this, DI.CB)) { + *this, DI.CB, + UsedAssumedInformation)) { LLVM_DEBUG( dbgs() << "[H2S] Unexpected failure in getAssumedUnderlyingObjects!\n"); @@ -6239,6 +6497,8 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { dbgs() << "[H2S] unique free call might free unknown allocations\n"); return false; } + if (DI->PotentialAllocationCalls.empty()) + return true; if (DI->PotentialAllocationCalls.size() > 1) { LLVM_DEBUG(dbgs() << "[H2S] unique free call might free " << DI->PotentialAllocationCalls.size() @@ -6316,7 +6576,7 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { if (ValidUsesOnly && AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared) - A.emitRemark<OptimizationRemarkMissed>(AI.CB, "OMP113", Remark); + A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark); LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n"); ValidUsesOnly = false; @@ -6348,7 +6608,8 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { continue; if (Value *Align = getAllocAlignment(AI.CB, TLI)) { - if (!getAPInt(A, *this, *Align)) { + Optional<APInt> APAlign = getAPInt(A, *this, *Align); + if (!APAlign) { // Can't generate an alloca which respects the required alignment // on the allocation. LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB @@ -6356,14 +6617,23 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { AI.Status = AllocationInfo::INVALID; Changed = ChangeStatus::CHANGED; continue; + } else { + if (APAlign->ugt(llvm::Value::MaximumAlignment) || + !APAlign->isPowerOf2()) { + LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign + << "\n"); + AI.Status = AllocationInfo::INVALID; + Changed = ChangeStatus::CHANGED; + continue; + } } } if (MaxHeapToStackSize != -1) { Optional<APInt> Size = getSize(A, *this, AI); - if (!Size.hasValue() || Size.getValue().ugt(MaxHeapToStackSize)) { + if (!Size || Size.getValue().ugt(MaxHeapToStackSize)) { LLVM_DEBUG({ - if (!Size.hasValue()) + if (!Size) dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n"; else dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. " @@ -6395,8 +6665,10 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) { return Changed; } +} // namespace /// ----------------------- Privatizable Pointers ------------------------------ +namespace { struct AAPrivatizablePtrImpl : public AAPrivatizablePtr { AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A) : AAPrivatizablePtr(IRP, A), PrivatizableType(llvm::None) {} @@ -6414,9 +6686,9 @@ struct AAPrivatizablePtrImpl : public AAPrivatizablePtr { /// Return a privatizable type that encloses both T0 and T1. /// TODO: This is merely a stub for now as we should manage a mapping as well. Optional<Type *> combineTypes(Optional<Type *> T0, Optional<Type *> T1) { - if (!T0.hasValue()) + if (!T0) return T1; - if (!T1.hasValue()) + if (!T1) return T0; if (T0 == T1) return T0; @@ -6445,11 +6717,13 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Optional<Type *> identifyPrivatizableType(Attributor &A) override { // If this is a byval argument and we know all the call sites (so we can // rewrite them), there is no need to check them explicitly. - bool AllCallSitesKnown; - if (getIRPosition().hasAttr(Attribute::ByVal) && + bool UsedAssumedInformation = false; + SmallVector<Attribute, 1> Attrs; + getAttrs({Attribute::ByVal}, Attrs, /* IgnoreSubsumingPositions */ true); + if (!Attrs.empty() && A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this, - true, AllCallSitesKnown)) - return getAssociatedValue().getType()->getPointerElementType(); + true, UsedAssumedInformation)) + return Attrs[0].getValueAsType(); Optional<Type *> Ty; unsigned ArgNo = getIRPosition().getCallSiteArgNo(); @@ -6474,9 +6748,9 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { LLVM_DEBUG({ dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: "; - if (CSTy.hasValue() && CSTy.getValue()) + if (CSTy && CSTy.getValue()) CSTy.getValue()->print(dbgs()); - else if (CSTy.hasValue()) + else if (CSTy) dbgs() << "<nullptr>"; else dbgs() << "<none>"; @@ -6486,19 +6760,20 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { LLVM_DEBUG({ dbgs() << " : New Type: "; - if (Ty.hasValue() && Ty.getValue()) + if (Ty && Ty.getValue()) Ty.getValue()->print(dbgs()); - else if (Ty.hasValue()) + else if (Ty) dbgs() << "<nullptr>"; else dbgs() << "<none>"; dbgs() << "\n"; }); - return !Ty.hasValue() || Ty.getValue(); + return !Ty || Ty.getValue(); }; - if (!A.checkForAllCallSites(CallSiteCheck, *this, true, AllCallSitesKnown)) + if (!A.checkForAllCallSites(CallSiteCheck, *this, true, + UsedAssumedInformation)) return nullptr; return Ty; } @@ -6506,7 +6781,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { /// See AbstractAttribute::updateImpl(...). ChangeStatus updateImpl(Attributor &A) override { PrivatizableType = identifyPrivatizableType(A); - if (!PrivatizableType.hasValue()) + if (!PrivatizableType) return ChangeStatus::UNCHANGED; if (!PrivatizableType.getValue()) return indicatePessimisticFixpoint(); @@ -6518,8 +6793,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { // Avoid arguments with padding for now. if (!getIRPosition().hasAttr(Attribute::ByVal) && - !ArgumentPromotionPass::isDenselyPacked(PrivatizableType.getValue(), - A.getInfoCache().getDL())) { + !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) { LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n"); return indicatePessimisticFixpoint(); } @@ -6527,7 +6801,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { // Collect the types that will replace the privatizable type in the function // signature. SmallVector<Type *, 16> ReplacementTypes; - identifyReplacementTypes(PrivatizableType.getValue(), ReplacementTypes); + identifyReplacementTypes(*PrivatizableType, ReplacementTypes); // Verify callee and caller agree on how the promoted argument would be // passed. @@ -6545,9 +6819,9 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { return TTI->areTypesABICompatible( CB->getCaller(), CB->getCalledFunction(), ReplacementTypes); }; - bool AllCallSitesKnown; + bool UsedAssumedInformation = false; if (!A.checkForAllCallSites(CallSiteCheck, *this, true, - AllCallSitesKnown)) { + UsedAssumedInformation)) { LLVM_DEBUG( dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for " << Fn.getName() << "\n"); @@ -6595,7 +6869,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED); if (CBArgPrivAA.isValidState()) { auto CBArgPrivTy = CBArgPrivAA.getPrivatizableType(); - if (!CBArgPrivTy.hasValue()) + if (!CBArgPrivTy) continue; if (CBArgPrivTy.getValue() == PrivatizableType) continue; @@ -6642,7 +6916,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { DepClassTy::REQUIRED); if (DCArgPrivAA.isValidState()) { auto DCArgPrivTy = DCArgPrivAA.getPrivatizableType(); - if (!DCArgPrivTy.hasValue()) + if (!DCArgPrivTy) return true; if (DCArgPrivTy.getValue() == PrivatizableType) return true; @@ -6674,7 +6948,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { }; if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true, - AllCallSitesKnown)) + UsedAssumedInformation)) return indicatePessimisticFixpoint(); return ChangeStatus::UNCHANGED; @@ -6749,8 +7023,8 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Type *PrivPtrType = PrivType->getPointerTo(); if (Base->getType() != PrivPtrType) - Base = BitCastInst::CreateBitOrPointerCast(Base, PrivPtrType, "", - ACS.getInstruction()); + Base = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( + Base, PrivPtrType, "", ACS.getInstruction()); // Traverse the type, build GEPs and loads. if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) { @@ -6784,7 +7058,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { /// See AbstractAttribute::manifest(...) ChangeStatus manifest(Attributor &A) override { - if (!PrivatizableType.hasValue()) + if (!PrivatizableType) return ChangeStatus::UNCHANGED; assert(PrivatizableType.getValue() && "Expected privatizable type!"); @@ -6817,14 +7091,16 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Function &ReplacementFn, Function::arg_iterator ArgIt) { BasicBlock &EntryBB = ReplacementFn.getEntryBlock(); Instruction *IP = &*EntryBB.getFirstInsertionPt(); - Instruction *AI = new AllocaInst(PrivatizableType.getValue(), 0, + const DataLayout &DL = IP->getModule()->getDataLayout(); + unsigned AS = DL.getAllocaAddrSpace(); + Instruction *AI = new AllocaInst(PrivatizableType.getValue(), AS, Arg->getName() + ".priv", IP); createInitialization(PrivatizableType.getValue(), *AI, ReplacementFn, ArgIt->getArgNo(), *IP); if (AI->getType() != Arg->getType()) - AI = - BitCastInst::CreateBitOrPointerCast(AI, Arg->getType(), "", IP); + AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( + AI, Arg->getType(), "", IP); Arg->replaceAllUsesWith(AI); for (CallInst *CI : TailCalls) @@ -6841,8 +7117,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { // When no alignment is specified for the load instruction, // natural alignment is assumed. createReplacementValues( - assumeAligned(AlignAA.getAssumedAlign()), - PrivatizableType.getValue(), ACS, + AlignAA.getAssumedAlign(), *PrivatizableType, ACS, ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()), NewArgOperands); }; @@ -6850,7 +7125,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { // Collect the types that will replace the privatizable type in the function // signature. SmallVector<Type *, 16> ReplacementTypes; - identifyReplacementTypes(PrivatizableType.getValue(), ReplacementTypes); + identifyReplacementTypes(*PrivatizableType, ReplacementTypes); // Register a rewrite of the argument. if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes, @@ -6897,7 +7172,7 @@ struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl { auto &PrivArgAA = A.getAAFor<AAPrivatizablePtr>( *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED); if (PrivArgAA.isAssumedPrivatizablePtr()) - return Obj->getType()->getPointerElementType(); + return PrivArgAA.getPrivatizableType(); } LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid " @@ -6926,7 +7201,7 @@ struct AAPrivatizablePtrCallSiteArgument final /// See AbstractAttribute::updateImpl(...). ChangeStatus updateImpl(Attributor &A) override { PrivatizableType = identifyPrivatizableType(A); - if (!PrivatizableType.hasValue()) + if (!PrivatizableType) return ChangeStatus::UNCHANGED; if (!PrivatizableType.getValue()) return indicatePessimisticFixpoint(); @@ -6992,10 +7267,12 @@ struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating { STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr); } }; +} // namespace /// -------------------- Memory Behavior Attributes ---------------------------- /// Includes read-none, read-only, and write-only. /// ---------------------------------------------------------------------------- +namespace { struct AAMemoryBehaviorImpl : public AAMemoryBehavior { AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A) : AAMemoryBehavior(IRP, A) {} @@ -7495,6 +7772,7 @@ void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U, if (UserI->mayWriteToMemory()) removeAssumedBits(NO_WRITES); } +} // namespace /// -------------------- Memory Locations Attributes --------------------------- /// Includes read-none, argmemonly, inaccessiblememonly, @@ -7528,6 +7806,7 @@ std::string AAMemoryLocation::getMemoryLocationsAsStr( return S; } +namespace { struct AAMemoryLocationImpl : public AAMemoryLocation { AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A) @@ -7772,8 +8051,10 @@ void AAMemoryLocationImpl::categorizePtrValue( << getMemoryLocationsAsStr(State.getAssumed()) << "]\n"); SmallVector<Value *, 8> Objects; + bool UsedAssumedInformation = false; if (!AA::getAssumedUnderlyingObjects(A, Ptr, Objects, *this, &I, - /* Intraprocedural */ true)) { + UsedAssumedInformation, + AA::Intraprocedural)) { LLVM_DEBUG( dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n"); updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed, @@ -8042,9 +8323,11 @@ struct AAMemoryLocationCallSite final : AAMemoryLocationImpl { STATS_DECLTRACK_CS_ATTR(readnone) } }; +} // namespace /// ------------------ Value Constant Range Attribute ------------------------- +namespace { struct AAValueConstantRangeImpl : AAValueConstantRange { using StateType = IntegerRangeState; AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A) @@ -8379,7 +8662,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { const auto &SimplifiedLHS = A.getAssumedSimplified(IRPosition::value(*LHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedLHS.hasValue()) + if (!SimplifiedLHS) return true; if (!SimplifiedLHS.getValue()) return false; @@ -8388,7 +8671,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { const auto &SimplifiedRHS = A.getAssumedSimplified(IRPosition::value(*RHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedRHS.hasValue()) + if (!SimplifiedRHS) return true; if (!SimplifiedRHS.getValue()) return false; @@ -8432,7 +8715,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { const auto &SimplifiedOpV = A.getAssumedSimplified(IRPosition::value(*OpV, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedOpV.hasValue()) + if (!SimplifiedOpV) return true; if (!SimplifiedOpV.getValue()) return false; @@ -8462,7 +8745,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { const auto &SimplifiedLHS = A.getAssumedSimplified(IRPosition::value(*LHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedLHS.hasValue()) + if (!SimplifiedLHS) return true; if (!SimplifiedLHS.getValue()) return false; @@ -8471,7 +8754,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { const auto &SimplifiedRHS = A.getAssumedSimplified(IRPosition::value(*RHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedRHS.hasValue()) + if (!SimplifiedRHS) return true; if (!SimplifiedRHS.getValue()) return false; @@ -8536,7 +8819,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { const auto &SimplifiedOpV = A.getAssumedSimplified(IRPosition::value(V, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedOpV.hasValue()) + if (!SimplifiedOpV) return true; if (!SimplifiedOpV.getValue()) return false; @@ -8588,8 +8871,10 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl { IntegerRangeState T(getBitWidth()); + bool UsedAssumedInformation = false; if (!genericValueTraversal<IntegerRangeState>(A, getIRPosition(), *this, T, VisitValueCB, getCtxI(), + UsedAssumedInformation, /* UseValueSimplify */ false)) return indicatePessimisticFixpoint(); @@ -8683,21 +8968,23 @@ struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating { STATS_DECLTRACK_CSARG_ATTR(value_range) } }; +} // namespace /// ------------------ Potential Values Attribute ------------------------- -struct AAPotentialValuesImpl : AAPotentialValues { +namespace { +struct AAPotentialConstantValuesImpl : AAPotentialConstantValues { using StateType = PotentialConstantIntValuesState; - AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A) - : AAPotentialValues(IRP, A) {} + AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A) + : AAPotentialConstantValues(IRP, A) {} /// See AbstractAttribute::initialize(..). void initialize(Attributor &A) override { if (A.hasSimplificationCallback(getIRPosition())) indicatePessimisticFixpoint(); else - AAPotentialValues::initialize(A); + AAPotentialConstantValues::initialize(A); } /// See AbstractAttribute::getAsStr(). @@ -8714,13 +9001,14 @@ struct AAPotentialValuesImpl : AAPotentialValues { } }; -struct AAPotentialValuesArgument final - : AAArgumentFromCallSiteArguments<AAPotentialValues, AAPotentialValuesImpl, +struct AAPotentialConstantValuesArgument final + : AAArgumentFromCallSiteArguments<AAPotentialConstantValues, + AAPotentialConstantValuesImpl, PotentialConstantIntValuesState> { - using Base = - AAArgumentFromCallSiteArguments<AAPotentialValues, AAPotentialValuesImpl, - PotentialConstantIntValuesState>; - AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A) + using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues, + AAPotentialConstantValuesImpl, + PotentialConstantIntValuesState>; + AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {} /// See AbstractAttribute::initialize(..). @@ -8738,11 +9026,12 @@ struct AAPotentialValuesArgument final } }; -struct AAPotentialValuesReturned - : AAReturnedFromReturnedValues<AAPotentialValues, AAPotentialValuesImpl> { - using Base = - AAReturnedFromReturnedValues<AAPotentialValues, AAPotentialValuesImpl>; - AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A) +struct AAPotentialConstantValuesReturned + : AAReturnedFromReturnedValues<AAPotentialConstantValues, + AAPotentialConstantValuesImpl> { + using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues, + AAPotentialConstantValuesImpl>; + AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {} /// See AbstractAttribute::trackStatistics() @@ -8751,13 +9040,13 @@ struct AAPotentialValuesReturned } }; -struct AAPotentialValuesFloating : AAPotentialValuesImpl { - AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A) - : AAPotentialValuesImpl(IRP, A) {} +struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl { + AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A) + : AAPotentialConstantValuesImpl(IRP, A) {} /// See AbstractAttribute::initialize(..). void initialize(Attributor &A) override { - AAPotentialValuesImpl::initialize(A); + AAPotentialConstantValuesImpl::initialize(A); if (isAtFixpoint()) return; @@ -8783,7 +9072,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { indicatePessimisticFixpoint(); - LLVM_DEBUG(dbgs() << "[AAPotentialValues] We give up: " + LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: " << getAssociatedValue() << "\n"); } @@ -8891,7 +9180,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedLHS = A.getAssumedSimplified(IRPosition::value(*LHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedLHS.hasValue()) + if (!SimplifiedLHS) return ChangeStatus::UNCHANGED; if (!SimplifiedLHS.getValue()) return indicatePessimisticFixpoint(); @@ -8900,7 +9189,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedRHS = A.getAssumedSimplified(IRPosition::value(*RHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedRHS.hasValue()) + if (!SimplifiedRHS) return ChangeStatus::UNCHANGED; if (!SimplifiedRHS.getValue()) return indicatePessimisticFixpoint(); @@ -8909,18 +9198,18 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy()) return indicatePessimisticFixpoint(); - auto &LHSAA = A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*LHS), - DepClassTy::REQUIRED); + auto &LHSAA = A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*LHS), DepClassTy::REQUIRED); if (!LHSAA.isValidState()) return indicatePessimisticFixpoint(); - auto &RHSAA = A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*RHS), - DepClassTy::REQUIRED); + auto &RHSAA = A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*RHS), DepClassTy::REQUIRED); if (!RHSAA.isValidState()) return indicatePessimisticFixpoint(); - const DenseSet<APInt> &LHSAAPVS = LHSAA.getAssumedSet(); - const DenseSet<APInt> &RHSAAPVS = RHSAA.getAssumedSet(); + const SetTy &LHSAAPVS = LHSAA.getAssumedSet(); + const SetTy &RHSAAPVS = RHSAA.getAssumedSet(); // TODO: make use of undef flag to limit potential values aggressively. bool MaybeTrue = false, MaybeFalse = false; @@ -8974,7 +9263,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedLHS = A.getAssumedSimplified(IRPosition::value(*LHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedLHS.hasValue()) + if (!SimplifiedLHS) return ChangeStatus::UNCHANGED; if (!SimplifiedLHS.getValue()) return indicatePessimisticFixpoint(); @@ -8983,7 +9272,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedRHS = A.getAssumedSimplified(IRPosition::value(*RHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedRHS.hasValue()) + if (!SimplifiedRHS) return ChangeStatus::UNCHANGED; if (!SimplifiedRHS.getValue()) return indicatePessimisticFixpoint(); @@ -8997,21 +9286,21 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { // Check if we only need one operand. bool OnlyLeft = false, OnlyRight = false; - if (C.hasValue() && *C && (*C)->isOneValue()) + if (C && *C && (*C)->isOneValue()) OnlyLeft = true; - else if (C.hasValue() && *C && (*C)->isZeroValue()) + else if (C && *C && (*C)->isZeroValue()) OnlyRight = true; - const AAPotentialValues *LHSAA = nullptr, *RHSAA = nullptr; + const AAPotentialConstantValues *LHSAA = nullptr, *RHSAA = nullptr; if (!OnlyRight) { - LHSAA = &A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*LHS), - DepClassTy::REQUIRED); + LHSAA = &A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*LHS), DepClassTy::REQUIRED); if (!LHSAA->isValidState()) return indicatePessimisticFixpoint(); } if (!OnlyLeft) { - RHSAA = &A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*RHS), - DepClassTy::REQUIRED); + RHSAA = &A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*RHS), DepClassTy::REQUIRED); if (!RHSAA->isValidState()) return indicatePessimisticFixpoint(); } @@ -9049,17 +9338,17 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedSrc = A.getAssumedSimplified(IRPosition::value(*Src, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedSrc.hasValue()) + if (!SimplifiedSrc) return ChangeStatus::UNCHANGED; if (!SimplifiedSrc.getValue()) return indicatePessimisticFixpoint(); Src = *SimplifiedSrc; - auto &SrcAA = A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*Src), - DepClassTy::REQUIRED); + auto &SrcAA = A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*Src), DepClassTy::REQUIRED); if (!SrcAA.isValidState()) return indicatePessimisticFixpoint(); - const DenseSet<APInt> &SrcAAPVS = SrcAA.getAssumedSet(); + const SetTy &SrcAAPVS = SrcAA.getAssumedSet(); if (SrcAA.undefIsContained()) unionAssumedWithUndef(); else { @@ -9082,7 +9371,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedLHS = A.getAssumedSimplified(IRPosition::value(*LHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedLHS.hasValue()) + if (!SimplifiedLHS) return ChangeStatus::UNCHANGED; if (!SimplifiedLHS.getValue()) return indicatePessimisticFixpoint(); @@ -9091,7 +9380,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedRHS = A.getAssumedSimplified(IRPosition::value(*RHS, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedRHS.hasValue()) + if (!SimplifiedRHS) return ChangeStatus::UNCHANGED; if (!SimplifiedRHS.getValue()) return indicatePessimisticFixpoint(); @@ -9100,18 +9389,18 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy()) return indicatePessimisticFixpoint(); - auto &LHSAA = A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*LHS), - DepClassTy::REQUIRED); + auto &LHSAA = A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*LHS), DepClassTy::REQUIRED); if (!LHSAA.isValidState()) return indicatePessimisticFixpoint(); - auto &RHSAA = A.getAAFor<AAPotentialValues>(*this, IRPosition::value(*RHS), - DepClassTy::REQUIRED); + auto &RHSAA = A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(*RHS), DepClassTy::REQUIRED); if (!RHSAA.isValidState()) return indicatePessimisticFixpoint(); - const DenseSet<APInt> &LHSAAPVS = LHSAA.getAssumedSet(); - const DenseSet<APInt> &RHSAAPVS = RHSAA.getAssumedSet(); + const SetTy &LHSAAPVS = LHSAA.getAssumedSet(); + const SetTy &RHSAAPVS = RHSAA.getAssumedSet(); const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0); // TODO: make use of undef flag to limit potential values aggressively. @@ -9150,13 +9439,13 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { const auto &SimplifiedIncomingValue = A.getAssumedSimplified( IRPosition::value(*IncomingValue, getCallBaseContext()), *this, UsedAssumedInformation); - if (!SimplifiedIncomingValue.hasValue()) + if (!SimplifiedIncomingValue) continue; if (!SimplifiedIncomingValue.getValue()) return indicatePessimisticFixpoint(); IncomingValue = *SimplifiedIncomingValue; - auto &PotentialValuesAA = A.getAAFor<AAPotentialValues>( + auto &PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>( *this, IRPosition::value(*IncomingValue), DepClassTy::REQUIRED); if (!PotentialValuesAA.isValidState()) return indicatePessimisticFixpoint(); @@ -9169,30 +9458,6 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { : ChangeStatus::CHANGED; } - ChangeStatus updateWithLoad(Attributor &A, LoadInst &L) { - if (!L.getType()->isIntegerTy()) - return indicatePessimisticFixpoint(); - - auto Union = [&](Value &V) { - if (isa<UndefValue>(V)) { - unionAssumedWithUndef(); - return true; - } - if (ConstantInt *CI = dyn_cast<ConstantInt>(&V)) { - unionAssumed(CI->getValue()); - return true; - } - return false; - }; - auto AssumedBefore = getAssumed(); - - if (!AAValueSimplifyImpl::handleLoad(A, *this, L, Union)) - return indicatePessimisticFixpoint(); - - return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED - : ChangeStatus::CHANGED; - } - /// See AbstractAttribute::updateImpl(...). ChangeStatus updateImpl(Attributor &A) override { Value &V = getAssociatedValue(); @@ -9213,9 +9478,6 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { if (auto *PHI = dyn_cast<PHINode>(I)) return updateWithPHINode(A, PHI); - if (auto *L = dyn_cast<LoadInst>(I)) - return updateWithLoad(A, *L); - return indicatePessimisticFixpoint(); } @@ -9225,14 +9487,15 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl { } }; -struct AAPotentialValuesFunction : AAPotentialValuesImpl { - AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A) - : AAPotentialValuesImpl(IRP, A) {} +struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl { + AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A) + : AAPotentialConstantValuesImpl(IRP, A) {} /// See AbstractAttribute::initialize(...). ChangeStatus updateImpl(Attributor &A) override { - llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will " - "not be called"); + llvm_unreachable( + "AAPotentialConstantValues(Function|CallSite)::updateImpl will " + "not be called"); } /// See AbstractAttribute::trackStatistics() @@ -9241,9 +9504,9 @@ struct AAPotentialValuesFunction : AAPotentialValuesImpl { } }; -struct AAPotentialValuesCallSite : AAPotentialValuesFunction { - AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A) - : AAPotentialValuesFunction(IRP, A) {} +struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction { + AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A) + : AAPotentialConstantValuesFunction(IRP, A) {} /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { @@ -9251,11 +9514,13 @@ struct AAPotentialValuesCallSite : AAPotentialValuesFunction { } }; -struct AAPotentialValuesCallSiteReturned - : AACallSiteReturnedFromReturned<AAPotentialValues, AAPotentialValuesImpl> { - AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A) - : AACallSiteReturnedFromReturned<AAPotentialValues, - AAPotentialValuesImpl>(IRP, A) {} +struct AAPotentialConstantValuesCallSiteReturned + : AACallSiteReturnedFromReturned<AAPotentialConstantValues, + AAPotentialConstantValuesImpl> { + AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP, + Attributor &A) + : AACallSiteReturnedFromReturned<AAPotentialConstantValues, + AAPotentialConstantValuesImpl>(IRP, A) {} /// See AbstractAttribute::trackStatistics() void trackStatistics() const override { @@ -9263,13 +9528,15 @@ struct AAPotentialValuesCallSiteReturned } }; -struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating { - AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A) - : AAPotentialValuesFloating(IRP, A) {} +struct AAPotentialConstantValuesCallSiteArgument + : AAPotentialConstantValuesFloating { + AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP, + Attributor &A) + : AAPotentialConstantValuesFloating(IRP, A) {} /// See AbstractAttribute::initialize(..). void initialize(Attributor &A) override { - AAPotentialValuesImpl::initialize(A); + AAPotentialConstantValuesImpl::initialize(A); if (isAtFixpoint()) return; @@ -9292,8 +9559,8 @@ struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating { ChangeStatus updateImpl(Attributor &A) override { Value &V = getAssociatedValue(); auto AssumedBefore = getAssumed(); - auto &AA = A.getAAFor<AAPotentialValues>(*this, IRPosition::value(V), - DepClassTy::REQUIRED); + auto &AA = A.getAAFor<AAPotentialConstantValues>( + *this, IRPosition::value(V), DepClassTy::REQUIRED); const auto &S = AA.getAssumed(); unionAssumed(S); return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED @@ -9365,7 +9632,7 @@ struct AANoUndefImpl : AANoUndef { // considered to be dead. We don't manifest noundef in such positions for // the same reason above. if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation) - .hasValue()) + .has_value()) return ChangeStatus::UNCHANGED; return AANoUndef::manifest(A); } @@ -9400,8 +9667,10 @@ struct AANoUndefFloating : public AANoUndefImpl { }; StateType T; + bool UsedAssumedInformation = false; if (!genericValueTraversal<StateType>(A, getIRPosition(), *this, T, - VisitValueCB, getCtxI())) + VisitValueCB, getCtxI(), + UsedAssumedInformation)) return indicatePessimisticFixpoint(); return clampStateAndIndicateChange(getState(), T); @@ -9518,9 +9787,10 @@ struct AACallEdgesCallSite : public AACallEdgesImpl { // Process any value that we might call. auto ProcessCalledOperand = [&](Value *V) { bool DummyValue = false; + bool UsedAssumedInformation = false; if (!genericValueTraversal<bool>(A, IRPosition::value(*V), *this, DummyValue, VisitValue, nullptr, - false)) { + UsedAssumedInformation, false)) { // If we haven't gone through all values, assume that there are unknown // callees. setHasUnknownCallee(true, Change); @@ -9530,7 +9800,9 @@ struct AACallEdgesCallSite : public AACallEdgesImpl { CallBase *CB = cast<CallBase>(getCtxI()); if (CB->isInlineAsm()) { - setHasUnknownCallee(false, Change); + if (!hasAssumption(*CB->getCaller(), "ompx_no_call_asm") && + !hasAssumption(*CB, "ompx_no_call_asm")) + setHasUnknownCallee(false, Change); return Change; } @@ -9584,7 +9856,8 @@ struct AACallEdgesFunction : public AACallEdgesImpl { // Visit all callable instructions. bool UsedAssumedInformation = false; if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this, - UsedAssumedInformation)) { + UsedAssumedInformation, + /* CheckBBLivenessOnly */ true)) { // If we haven't looked at all call like instructions, assume that there // are unknown callees. setHasUnknownCallee(true, Change); @@ -9656,7 +9929,7 @@ private: ArrayRef<const AACallEdges *> AAEdgesList, const Function &Fn) { Optional<bool> Cached = isCachedReachable(Fn); - if (Cached.hasValue()) + if (Cached) return Cached.getValue(); // The query was not cached, thus it is new. We need to request an update @@ -9691,6 +9964,10 @@ private: const SetVector<Function *> &Edges = AAEdges->getOptimisticEdges(); for (Function *Edge : Edges) { + // Functions that do not call back into the module can be ignored. + if (Edge->hasFnAttribute(Attribute::NoCallback)) + continue; + // We don't need a dependency if the result is reachable. const AAFunctionReachability &EdgeReachability = A.getAAFor<AAFunctionReachability>( @@ -9820,22 +10097,21 @@ public: } // Update the Instruction queries. - const AAReachability *Reachability; if (!InstQueries.empty()) { - Reachability = &A.getAAFor<AAReachability>( + const AAReachability *Reachability = &A.getAAFor<AAReachability>( *this, IRPosition::function(*getAssociatedFunction()), DepClassTy::REQUIRED); - } - // Check for local callbases first. - for (auto &InstPair : InstQueries) { - SmallVector<const AACallEdges *> CallEdges; - bool AllKnown = - getReachableCallEdges(A, *Reachability, *InstPair.first, CallEdges); - // Update will return change if we this effects any queries. - if (!AllKnown) - InstPair.second.CanReachUnknownCallee = true; - Change |= InstPair.second.update(A, *this, CallEdges); + // Check for local callbases first. + for (auto &InstPair : InstQueries) { + SmallVector<const AACallEdges *> CallEdges; + bool AllKnown = + getReachableCallEdges(A, *Reachability, *InstPair.first, CallEdges); + // Update will return change if we this effects any queries. + if (!AllKnown) + InstPair.second.CanReachUnknownCallee = true; + Change |= InstPair.second.update(A, *this, CallEdges); + } } return Change; @@ -9862,13 +10138,15 @@ private: /// Used to answer if a call base inside this function can reach a specific /// function. - DenseMap<const CallBase *, QueryResolver> CBQueries; + MapVector<const CallBase *, QueryResolver> CBQueries; /// This is for instruction queries than scan "forward". - DenseMap<const Instruction *, QueryResolver> InstQueries; + MapVector<const Instruction *, QueryResolver> InstQueries; }; +} // namespace /// ---------------------- Assumption Propagation ------------------------------ +namespace { struct AAAssumptionInfoImpl : public AAAssumptionInfo { AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A, const DenseSet<StringRef> &Known) @@ -9938,12 +10216,13 @@ struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl { return !getAssumed().empty() || !getKnown().empty(); }; - bool AllCallSitesKnown; + bool UsedAssumedInformation = false; // Get the intersection of all assumptions held by this node's predecessors. // If we don't know all the call sites then this is either an entry into the // call graph or an empty node. This node is known to only contain its own // assumptions and can be propagated to its successors. - if (!A.checkForAllCallSites(CallSitePred, *this, true, AllCallSitesKnown)) + if (!A.checkForAllCallSites(CallSitePred, *this, true, + UsedAssumedInformation)) return indicatePessimisticFixpoint(); return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED; @@ -10001,6 +10280,7 @@ private: return Assumptions; } }; +} // namespace AACallGraphNode *AACallEdgeIterator::operator*() const { return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>( @@ -10023,6 +10303,7 @@ const char AANoReturn::ID = 0; const char AAIsDead::ID = 0; const char AADereferenceable::ID = 0; const char AAAlign::ID = 0; +const char AAInstanceInfo::ID = 0; const char AANoCapture::ID = 0; const char AAValueSimplify::ID = 0; const char AAHeapToStack::ID = 0; @@ -10030,7 +10311,7 @@ const char AAPrivatizablePtr::ID = 0; const char AAMemoryBehavior::ID = 0; const char AAMemoryLocation::ID = 0; const char AAValueConstantRange::ID = 0; -const char AAPotentialValues::ID = 0; +const char AAPotentialConstantValues::ID = 0; const char AANoUndef::ID = 0; const char AACallEdges::ID = 0; const char AAFunctionReachability::ID = 0; @@ -10145,9 +10426,10 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign) +CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange) -CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues) +CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef) CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo) |