diff options
Diffstat (limited to 'llvm/lib/Analysis/CGSCCPassManager.cpp')
| -rw-r--r-- | llvm/lib/Analysis/CGSCCPassManager.cpp | 709 |
1 files changed, 709 insertions, 0 deletions
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp new file mode 100644 index 000000000000..a0b3f83cca6a --- /dev/null +++ b/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -0,0 +1,709 @@ +//===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <iterator> + +#define DEBUG_TYPE "cgscc" + +using namespace llvm; + +// Explicit template instantiations and specialization definitions for core +// template typedefs. +namespace llvm { + +// Explicit instantiations for the core proxy templates. +template class AllAnalysesOn<LazyCallGraph::SCC>; +template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>; +template class PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, + LazyCallGraph &, CGSCCUpdateResult &>; +template class InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>; +template class OuterAnalysisManagerProxy<ModuleAnalysisManager, + LazyCallGraph::SCC, LazyCallGraph &>; +template class OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>; + +/// Explicitly specialize the pass manager run method to handle call graph +/// updates. +template <> +PreservedAnalyses +PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, + CGSCCUpdateResult &>::run(LazyCallGraph::SCC &InitialC, + CGSCCAnalysisManager &AM, + LazyCallGraph &G, CGSCCUpdateResult &UR) { + // Request PassInstrumentation from analysis manager, will use it to run + // instrumenting callbacks for the passes later. + PassInstrumentation PI = + AM.getResult<PassInstrumentationAnalysis>(InitialC, G); + + PreservedAnalyses PA = PreservedAnalyses::all(); + + if (DebugLogging) + dbgs() << "Starting CGSCC pass manager run.\n"; + + // The SCC may be refined while we are running passes over it, so set up + // a pointer that we can update. + LazyCallGraph::SCC *C = &InitialC; + + for (auto &Pass : Passes) { + if (DebugLogging) + dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n"; + + // Check the PassInstrumentation's BeforePass callbacks before running the + // pass, skip its execution completely if asked to (callback returns false). + if (!PI.runBeforePass(*Pass, *C)) + continue; + + PreservedAnalyses PassPA = Pass->run(*C, AM, G, UR); + + if (UR.InvalidatedSCCs.count(C)) + PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass); + else + PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C); + + // Update the SCC if necessary. + C = UR.UpdatedC ? UR.UpdatedC : C; + + // If the CGSCC pass wasn't able to provide a valid updated SCC, the + // current SCC may simply need to be skipped if invalid. + if (UR.InvalidatedSCCs.count(C)) { + LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n"); + break; + } + // Check that we didn't miss any update scenario. + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + + // Update the analysis manager as each pass runs and potentially + // invalidates analyses. + AM.invalidate(*C, PassPA); + + // Finally, we intersect the final preserved analyses to compute the + // aggregate preserved set for this pass manager. + PA.intersect(std::move(PassPA)); + + // FIXME: Historically, the pass managers all called the LLVM context's + // yield function here. We don't have a generic way to acquire the + // context and it isn't yet clear what the right pattern is for yielding + // in the new pass manager so it is currently omitted. + // ...getContext().yield(); + } + + // Before we mark all of *this* SCC's analyses as preserved below, intersect + // this with the cross-SCC preserved analysis set. This is used to allow + // CGSCC passes to mutate ancestor SCCs and still trigger proper invalidation + // for them. + UR.CrossSCCPA.intersect(PA); + + // Invalidation was handled after each pass in the above loop for the current + // SCC. Therefore, the remaining analysis results in the AnalysisManager are + // preserved. We mark this with a set so that we don't need to inspect each + // one individually. + PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>(); + + if (DebugLogging) + dbgs() << "Finished CGSCC pass manager run.\n"; + + return PA; +} + +bool CGSCCAnalysisManagerModuleProxy::Result::invalidate( + Module &M, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &Inv) { + // If literally everything is preserved, we're done. + if (PA.areAllPreserved()) + return false; // This is still a valid proxy. + + // If this proxy or the call graph is going to be invalidated, we also need + // to clear all the keys coming from that analysis. + // + // We also directly invalidate the FAM's module proxy if necessary, and if + // that proxy isn't preserved we can't preserve this proxy either. We rely on + // it to handle module -> function analysis invalidation in the face of + // structural changes and so if it's unavailable we conservatively clear the + // entire SCC layer as well rather than trying to do invalidation ourselves. + auto PAC = PA.getChecker<CGSCCAnalysisManagerModuleProxy>(); + if (!(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()) || + Inv.invalidate<LazyCallGraphAnalysis>(M, PA) || + Inv.invalidate<FunctionAnalysisManagerModuleProxy>(M, PA)) { + InnerAM->clear(); + + // And the proxy itself should be marked as invalid so that we can observe + // the new call graph. This isn't strictly necessary because we cheat + // above, but is still useful. + return true; + } + + // Directly check if the relevant set is preserved so we can short circuit + // invalidating SCCs below. + bool AreSCCAnalysesPreserved = + PA.allAnalysesInSetPreserved<AllAnalysesOn<LazyCallGraph::SCC>>(); + + // Ok, we have a graph, so we can propagate the invalidation down into it. + G->buildRefSCCs(); + for (auto &RC : G->postorder_ref_sccs()) + for (auto &C : RC) { + Optional<PreservedAnalyses> InnerPA; + + // Check to see whether the preserved set needs to be adjusted based on + // module-level analysis invalidation triggering deferred invalidation + // for this SCC. + if (auto *OuterProxy = + InnerAM->getCachedResult<ModuleAnalysisManagerCGSCCProxy>(C)) + for (const auto &OuterInvalidationPair : + OuterProxy->getOuterInvalidations()) { + AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; + const auto &InnerAnalysisIDs = OuterInvalidationPair.second; + if (Inv.invalidate(OuterAnalysisID, M, PA)) { + if (!InnerPA) + InnerPA = PA; + for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) + InnerPA->abandon(InnerAnalysisID); + } + } + + // Check if we needed a custom PA set. If so we'll need to run the inner + // invalidation. + if (InnerPA) { + InnerAM->invalidate(C, *InnerPA); + continue; + } + + // Otherwise we only need to do invalidation if the original PA set didn't + // preserve all SCC analyses. + if (!AreSCCAnalysesPreserved) + InnerAM->invalidate(C, PA); + } + + // Return false to indicate that this result is still a valid proxy. + return false; +} + +template <> +CGSCCAnalysisManagerModuleProxy::Result +CGSCCAnalysisManagerModuleProxy::run(Module &M, ModuleAnalysisManager &AM) { + // Force the Function analysis manager to also be available so that it can + // be accessed in an SCC analysis and proxied onward to function passes. + // FIXME: It is pretty awkward to just drop the result here and assert that + // we can find it again later. + (void)AM.getResult<FunctionAnalysisManagerModuleProxy>(M); + + return Result(*InnerAM, AM.getResult<LazyCallGraphAnalysis>(M)); +} + +AnalysisKey FunctionAnalysisManagerCGSCCProxy::Key; + +FunctionAnalysisManagerCGSCCProxy::Result +FunctionAnalysisManagerCGSCCProxy::run(LazyCallGraph::SCC &C, + CGSCCAnalysisManager &AM, + LazyCallGraph &CG) { + // Collect the FunctionAnalysisManager from the Module layer and use that to + // build the proxy result. + // + // This allows us to rely on the FunctionAnalysisMangaerModuleProxy to + // invalidate the function analyses. + auto &MAM = AM.getResult<ModuleAnalysisManagerCGSCCProxy>(C, CG).getManager(); + Module &M = *C.begin()->getFunction().getParent(); + auto *FAMProxy = MAM.getCachedResult<FunctionAnalysisManagerModuleProxy>(M); + assert(FAMProxy && "The CGSCC pass manager requires that the FAM module " + "proxy is run on the module prior to entering the CGSCC " + "walk."); + + // Note that we special-case invalidation handling of this proxy in the CGSCC + // analysis manager's Module proxy. This avoids the need to do anything + // special here to recompute all of this if ever the FAM's module proxy goes + // away. + return Result(FAMProxy->getManager()); +} + +bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate( + LazyCallGraph::SCC &C, const PreservedAnalyses &PA, + CGSCCAnalysisManager::Invalidator &Inv) { + // If literally everything is preserved, we're done. + if (PA.areAllPreserved()) + return false; // This is still a valid proxy. + + // If this proxy isn't marked as preserved, then even if the result remains + // valid, the key itself may no longer be valid, so we clear everything. + // + // Note that in order to preserve this proxy, a module pass must ensure that + // the FAM has been completely updated to handle the deletion of functions. + // Specifically, any FAM-cached results for those functions need to have been + // forcibly cleared. When preserved, this proxy will only invalidate results + // cached on functions *still in the module* at the end of the module pass. + auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>(); + if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) { + for (LazyCallGraph::Node &N : C) + FAM->clear(N.getFunction(), N.getFunction().getName()); + + return true; + } + + // Directly check if the relevant set is preserved. + bool AreFunctionAnalysesPreserved = + PA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>(); + + // Now walk all the functions to see if any inner analysis invalidation is + // necessary. + for (LazyCallGraph::Node &N : C) { + Function &F = N.getFunction(); + Optional<PreservedAnalyses> FunctionPA; + + // Check to see whether the preserved set needs to be pruned based on + // SCC-level analysis invalidation that triggers deferred invalidation + // registered with the outer analysis manager proxy for this function. + if (auto *OuterProxy = + FAM->getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F)) + for (const auto &OuterInvalidationPair : + OuterProxy->getOuterInvalidations()) { + AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; + const auto &InnerAnalysisIDs = OuterInvalidationPair.second; + if (Inv.invalidate(OuterAnalysisID, C, PA)) { + if (!FunctionPA) + FunctionPA = PA; + for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) + FunctionPA->abandon(InnerAnalysisID); + } + } + + // Check if we needed a custom PA set, and if so we'll need to run the + // inner invalidation. + if (FunctionPA) { + FAM->invalidate(F, *FunctionPA); + continue; + } + + // Otherwise we only need to do invalidation if the original PA set didn't + // preserve all function analyses. + if (!AreFunctionAnalysesPreserved) + FAM->invalidate(F, PA); + } + + // Return false to indicate that this result is still a valid proxy. + return false; +} + +} // end namespace llvm + +/// When a new SCC is created for the graph and there might be function +/// analysis results cached for the functions now in that SCC two forms of +/// updates are required. +/// +/// First, a proxy from the SCC to the FunctionAnalysisManager needs to be +/// created so that any subsequent invalidation events to the SCC are +/// propagated to the function analysis results cached for functions within it. +/// +/// Second, if any of the functions within the SCC have analysis results with +/// outer analysis dependencies, then those dependencies would point to the +/// *wrong* SCC's analysis result. We forcibly invalidate the necessary +/// function analyses so that they don't retain stale handles. +static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C, + LazyCallGraph &G, + CGSCCAnalysisManager &AM) { + // Get the relevant function analysis manager. + auto &FAM = + AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, G).getManager(); + + // Now walk the functions in this SCC and invalidate any function analysis + // results that might have outer dependencies on an SCC analysis. + for (LazyCallGraph::Node &N : C) { + Function &F = N.getFunction(); + + auto *OuterProxy = + FAM.getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F); + if (!OuterProxy) + // No outer analyses were queried, nothing to do. + continue; + + // Forcibly abandon all the inner analyses with dependencies, but + // invalidate nothing else. + auto PA = PreservedAnalyses::all(); + for (const auto &OuterInvalidationPair : + OuterProxy->getOuterInvalidations()) { + const auto &InnerAnalysisIDs = OuterInvalidationPair.second; + for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) + PA.abandon(InnerAnalysisID); + } + + // Now invalidate anything we found. + FAM.invalidate(F, PA); + } +} + +/// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c +/// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly +/// added SCCs. +/// +/// The range of new SCCs must be in postorder already. The SCC they were split +/// out of must be provided as \p C. The current node being mutated and +/// triggering updates must be passed as \p N. +/// +/// This function returns the SCC containing \p N. This will be either \p C if +/// no new SCCs have been split out, or it will be the new SCC containing \p N. +template <typename SCCRangeT> +static LazyCallGraph::SCC * +incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G, + LazyCallGraph::Node &N, LazyCallGraph::SCC *C, + CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) { + using SCC = LazyCallGraph::SCC; + + if (NewSCCRange.begin() == NewSCCRange.end()) + return C; + + // Add the current SCC to the worklist as its shape has changed. + UR.CWorklist.insert(C); + LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist:" << *C + << "\n"); + + SCC *OldC = C; + + // Update the current SCC. Note that if we have new SCCs, this must actually + // change the SCC. + assert(C != &*NewSCCRange.begin() && + "Cannot insert new SCCs without changing current SCC!"); + C = &*NewSCCRange.begin(); + assert(G.lookupSCC(N) == C && "Failed to update current SCC!"); + + // If we had a cached FAM proxy originally, we will want to create more of + // them for each SCC that was split off. + bool NeedFAMProxy = + AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*OldC) != nullptr; + + // We need to propagate an invalidation call to all but the newly current SCC + // because the outer pass manager won't do that for us after splitting them. + // FIXME: We should accept a PreservedAnalysis from the CG updater so that if + // there are preserved analysis we can avoid invalidating them here for + // split-off SCCs. + // We know however that this will preserve any FAM proxy so go ahead and mark + // that. + PreservedAnalyses PA; + PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); + AM.invalidate(*OldC, PA); + + // Ensure the now-current SCC's function analyses are updated. + if (NeedFAMProxy) + updateNewSCCFunctionAnalyses(*C, G, AM); + + for (SCC &NewC : llvm::reverse(make_range(std::next(NewSCCRange.begin()), + NewSCCRange.end()))) { + assert(C != &NewC && "No need to re-visit the current SCC!"); + assert(OldC != &NewC && "Already handled the original SCC!"); + UR.CWorklist.insert(&NewC); + LLVM_DEBUG(dbgs() << "Enqueuing a newly formed SCC:" << NewC << "\n"); + + // Ensure new SCCs' function analyses are updated. + if (NeedFAMProxy) + updateNewSCCFunctionAnalyses(NewC, G, AM); + + // Also propagate a normal invalidation to the new SCC as only the current + // will get one from the pass manager infrastructure. + AM.invalidate(NewC, PA); + } + return C; +} + +LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass( + LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N, + CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) { + using Node = LazyCallGraph::Node; + using Edge = LazyCallGraph::Edge; + using SCC = LazyCallGraph::SCC; + using RefSCC = LazyCallGraph::RefSCC; + + RefSCC &InitialRC = InitialC.getOuterRefSCC(); + SCC *C = &InitialC; + RefSCC *RC = &InitialRC; + Function &F = N.getFunction(); + + // Walk the function body and build up the set of retained, promoted, and + // demoted edges. + SmallVector<Constant *, 16> Worklist; + SmallPtrSet<Constant *, 16> Visited; + SmallPtrSet<Node *, 16> RetainedEdges; + SmallSetVector<Node *, 4> PromotedRefTargets; + SmallSetVector<Node *, 4> DemotedCallTargets; + + // First walk the function and handle all called functions. We do this first + // because if there is a single call edge, whether there are ref edges is + // irrelevant. + for (Instruction &I : instructions(F)) + if (auto CS = CallSite(&I)) + if (Function *Callee = CS.getCalledFunction()) + if (Visited.insert(Callee).second && !Callee->isDeclaration()) { + Node &CalleeN = *G.lookup(*Callee); + Edge *E = N->lookup(CalleeN); + // FIXME: We should really handle adding new calls. While it will + // make downstream usage more complex, there is no fundamental + // limitation and it will allow passes within the CGSCC to be a bit + // more flexible in what transforms they can do. Until then, we + // verify that new calls haven't been introduced. + assert(E && "No function transformations should introduce *new* " + "call edges! Any new calls should be modeled as " + "promoted existing ref edges!"); + bool Inserted = RetainedEdges.insert(&CalleeN).second; + (void)Inserted; + assert(Inserted && "We should never visit a function twice."); + if (!E->isCall()) + PromotedRefTargets.insert(&CalleeN); + } + + // Now walk all references. + for (Instruction &I : instructions(F)) + for (Value *Op : I.operand_values()) + if (auto *C = dyn_cast<Constant>(Op)) + if (Visited.insert(C).second) + Worklist.push_back(C); + + auto VisitRef = [&](Function &Referee) { + Node &RefereeN = *G.lookup(Referee); + Edge *E = N->lookup(RefereeN); + // FIXME: Similarly to new calls, we also currently preclude + // introducing new references. See above for details. + assert(E && "No function transformations should introduce *new* ref " + "edges! Any new ref edges would require IPO which " + "function passes aren't allowed to do!"); + bool Inserted = RetainedEdges.insert(&RefereeN).second; + (void)Inserted; + assert(Inserted && "We should never visit a function twice."); + if (E->isCall()) + DemotedCallTargets.insert(&RefereeN); + }; + LazyCallGraph::visitReferences(Worklist, Visited, VisitRef); + + // Include synthetic reference edges to known, defined lib functions. + for (auto *F : G.getLibFunctions()) + // While the list of lib functions doesn't have repeats, don't re-visit + // anything handled above. + if (!Visited.count(F)) + VisitRef(*F); + + // First remove all of the edges that are no longer present in this function. + // The first step makes these edges uniformly ref edges and accumulates them + // into a separate data structure so removal doesn't invalidate anything. + SmallVector<Node *, 4> DeadTargets; + for (Edge &E : *N) { + if (RetainedEdges.count(&E.getNode())) + continue; + + SCC &TargetC = *G.lookupSCC(E.getNode()); + RefSCC &TargetRC = TargetC.getOuterRefSCC(); + if (&TargetRC == RC && E.isCall()) { + if (C != &TargetC) { + // For separate SCCs this is trivial. + RC->switchTrivialInternalEdgeToRef(N, E.getNode()); + } else { + // Now update the call graph. + C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, E.getNode()), + G, N, C, AM, UR); + } + } + + // Now that this is ready for actual removal, put it into our list. + DeadTargets.push_back(&E.getNode()); + } + // Remove the easy cases quickly and actually pull them out of our list. + DeadTargets.erase( + llvm::remove_if(DeadTargets, + [&](Node *TargetN) { + SCC &TargetC = *G.lookupSCC(*TargetN); + RefSCC &TargetRC = TargetC.getOuterRefSCC(); + + // We can't trivially remove internal targets, so skip + // those. + if (&TargetRC == RC) + return false; + + RC->removeOutgoingEdge(N, *TargetN); + LLVM_DEBUG(dbgs() << "Deleting outgoing edge from '" + << N << "' to '" << TargetN << "'\n"); + return true; + }), + DeadTargets.end()); + + // Now do a batch removal of the internal ref edges left. + auto NewRefSCCs = RC->removeInternalRefEdge(N, DeadTargets); + if (!NewRefSCCs.empty()) { + // The old RefSCC is dead, mark it as such. + UR.InvalidatedRefSCCs.insert(RC); + + // Note that we don't bother to invalidate analyses as ref-edge + // connectivity is not really observable in any way and is intended + // exclusively to be used for ordering of transforms rather than for + // analysis conclusions. + + // Update RC to the "bottom". + assert(G.lookupSCC(N) == C && "Changed the SCC when splitting RefSCCs!"); + RC = &C->getOuterRefSCC(); + assert(G.lookupRefSCC(N) == RC && "Failed to update current RefSCC!"); + + // The RC worklist is in reverse postorder, so we enqueue the new ones in + // RPO except for the one which contains the source node as that is the + // "bottom" we will continue processing in the bottom-up walk. + assert(NewRefSCCs.front() == RC && + "New current RefSCC not first in the returned list!"); + for (RefSCC *NewRC : llvm::reverse(make_range(std::next(NewRefSCCs.begin()), + NewRefSCCs.end()))) { + assert(NewRC != RC && "Should not encounter the current RefSCC further " + "in the postorder list of new RefSCCs."); + UR.RCWorklist.insert(NewRC); + LLVM_DEBUG(dbgs() << "Enqueuing a new RefSCC in the update worklist: " + << *NewRC << "\n"); + } + } + + // Next demote all the call edges that are now ref edges. This helps make + // the SCCs small which should minimize the work below as we don't want to + // form cycles that this would break. + for (Node *RefTarget : DemotedCallTargets) { + SCC &TargetC = *G.lookupSCC(*RefTarget); + RefSCC &TargetRC = TargetC.getOuterRefSCC(); + + // The easy case is when the target RefSCC is not this RefSCC. This is + // only supported when the target RefSCC is a child of this RefSCC. + if (&TargetRC != RC) { + assert(RC->isAncestorOf(TargetRC) && + "Cannot potentially form RefSCC cycles here!"); + RC->switchOutgoingEdgeToRef(N, *RefTarget); + LLVM_DEBUG(dbgs() << "Switch outgoing call edge to a ref edge from '" << N + << "' to '" << *RefTarget << "'\n"); + continue; + } + + // We are switching an internal call edge to a ref edge. This may split up + // some SCCs. + if (C != &TargetC) { + // For separate SCCs this is trivial. + RC->switchTrivialInternalEdgeToRef(N, *RefTarget); + continue; + } + + // Now update the call graph. + C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, *RefTarget), G, N, + C, AM, UR); + } + + // Now promote ref edges into call edges. + for (Node *CallTarget : PromotedRefTargets) { + SCC &TargetC = *G.lookupSCC(*CallTarget); + RefSCC &TargetRC = TargetC.getOuterRefSCC(); + + // The easy case is when the target RefSCC is not this RefSCC. This is + // only supported when the target RefSCC is a child of this RefSCC. + if (&TargetRC != RC) { + assert(RC->isAncestorOf(TargetRC) && + "Cannot potentially form RefSCC cycles here!"); + RC->switchOutgoingEdgeToCall(N, *CallTarget); + LLVM_DEBUG(dbgs() << "Switch outgoing ref edge to a call edge from '" << N + << "' to '" << *CallTarget << "'\n"); + continue; + } + LLVM_DEBUG(dbgs() << "Switch an internal ref edge to a call edge from '" + << N << "' to '" << *CallTarget << "'\n"); + + // Otherwise we are switching an internal ref edge to a call edge. This + // may merge away some SCCs, and we add those to the UpdateResult. We also + // need to make sure to update the worklist in the event SCCs have moved + // before the current one in the post-order sequence + bool HasFunctionAnalysisProxy = false; + auto InitialSCCIndex = RC->find(*C) - RC->begin(); + bool FormedCycle = RC->switchInternalEdgeToCall( + N, *CallTarget, [&](ArrayRef<SCC *> MergedSCCs) { + for (SCC *MergedC : MergedSCCs) { + assert(MergedC != &TargetC && "Cannot merge away the target SCC!"); + + HasFunctionAnalysisProxy |= + AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>( + *MergedC) != nullptr; + + // Mark that this SCC will no longer be valid. + UR.InvalidatedSCCs.insert(MergedC); + + // FIXME: We should really do a 'clear' here to forcibly release + // memory, but we don't have a good way of doing that and + // preserving the function analyses. + auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>(); + PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); + AM.invalidate(*MergedC, PA); + } + }); + + // If we formed a cycle by creating this call, we need to update more data + // structures. + if (FormedCycle) { + C = &TargetC; + assert(G.lookupSCC(N) == C && "Failed to update current SCC!"); + + // If one of the invalidated SCCs had a cached proxy to a function + // analysis manager, we need to create a proxy in the new current SCC as + // the invalidated SCCs had their functions moved. + if (HasFunctionAnalysisProxy) + AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G); + + // Any analyses cached for this SCC are no longer precise as the shape + // has changed by introducing this cycle. However, we have taken care to + // update the proxies so it remains valide. + auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>(); + PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); + AM.invalidate(*C, PA); + } + auto NewSCCIndex = RC->find(*C) - RC->begin(); + // If we have actually moved an SCC to be topologically "below" the current + // one due to merging, we will need to revisit the current SCC after + // visiting those moved SCCs. + // + // It is critical that we *do not* revisit the current SCC unless we + // actually move SCCs in the process of merging because otherwise we may + // form a cycle where an SCC is split apart, merged, split, merged and so + // on infinitely. + if (InitialSCCIndex < NewSCCIndex) { + // Put our current SCC back onto the worklist as we'll visit other SCCs + // that are now definitively ordered prior to the current one in the + // post-order sequence, and may end up observing more precise context to + // optimize the current SCC. + UR.CWorklist.insert(C); + LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist: " << *C + << "\n"); + // Enqueue in reverse order as we pop off the back of the worklist. + for (SCC &MovedC : llvm::reverse(make_range(RC->begin() + InitialSCCIndex, + RC->begin() + NewSCCIndex))) { + UR.CWorklist.insert(&MovedC); + LLVM_DEBUG(dbgs() << "Enqueuing a newly earlier in post-order SCC: " + << MovedC << "\n"); + } + } + } + + assert(!UR.InvalidatedSCCs.count(C) && "Invalidated the current SCC!"); + assert(!UR.InvalidatedRefSCCs.count(RC) && "Invalidated the current RefSCC!"); + assert(&C->getOuterRefSCC() == RC && "Current SCC not in current RefSCC!"); + + // Record the current RefSCC and SCC for higher layers of the CGSCC pass + // manager now that all the updates have been applied. + if (RC != &InitialRC) + UR.UpdatedRC = RC; + if (C != &InitialC) + UR.UpdatedC = C; + + return *C; +} |