summaryrefslogtreecommitdiff
path: root/lib/Transforms/Coroutines
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/Coroutines')
-rw-r--r--lib/Transforms/Coroutines/CoroCleanup.cpp1
-rw-r--r--lib/Transforms/Coroutines/CoroEarly.cpp3
-rw-r--r--lib/Transforms/Coroutines/CoroElide.cpp1
-rw-r--r--lib/Transforms/Coroutines/CoroFrame.cpp33
-rw-r--r--lib/Transforms/Coroutines/CoroSplit.cpp86
5 files changed, 100 insertions, 24 deletions
diff --git a/lib/Transforms/Coroutines/CoroCleanup.cpp b/lib/Transforms/Coroutines/CoroCleanup.cpp
index a97db6fde454e..5cf2a8c25d837 100644
--- a/lib/Transforms/Coroutines/CoroCleanup.cpp
+++ b/lib/Transforms/Coroutines/CoroCleanup.cpp
@@ -124,6 +124,7 @@ struct CoroCleanup : FunctionPass {
if (!L)
AU.setPreservesAll();
}
+ StringRef getPassName() const override { return "Coroutine Cleanup"; }
};
}
diff --git a/lib/Transforms/Coroutines/CoroEarly.cpp b/lib/Transforms/Coroutines/CoroEarly.cpp
index e8bb0ca99d8a5..b529891861651 100644
--- a/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -208,6 +208,9 @@ struct CoroEarly : public FunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
}
+ StringRef getPassName() const override {
+ return "Lower early coroutine intrinsics";
+ }
};
}
diff --git a/lib/Transforms/Coroutines/CoroElide.cpp b/lib/Transforms/Coroutines/CoroElide.cpp
index c6ac3f614ff7e..acb22449142b9 100644
--- a/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/lib/Transforms/Coroutines/CoroElide.cpp
@@ -301,6 +301,7 @@ struct CoroElide : FunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<AAResultsWrapperPass>();
}
+ StringRef getPassName() const override { return "Coroutine Elision"; }
};
}
diff --git a/lib/Transforms/Coroutines/CoroFrame.cpp b/lib/Transforms/Coroutines/CoroFrame.cpp
index 417d57f7625b4..85e9003ec3c56 100644
--- a/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -799,9 +799,9 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
splitAround(CSI, "CoroSuspend");
}
- // Put fallthrough CoroEnd into its own block. Note: Shape::buildFrom places
- // the fallthrough coro.end as the first element of CoroEnds array.
- splitAround(Shape.CoroEnds.front(), "CoroEnd");
+ // Put CoroEnds into their own blocks.
+ for (CoroEndInst *CE : Shape.CoroEnds)
+ splitAround(CE, "CoroEnd");
// Transforms multi-edge PHI Nodes, so that any value feeding into a PHI will
// never has its definition separated from the PHI by the suspend point.
@@ -813,19 +813,24 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
IRBuilder<> Builder(F.getContext());
SpillInfo Spills;
- // See if there are materializable instructions across suspend points.
- for (Instruction &I : instructions(F))
- if (materializable(I))
- for (User *U : I.users())
- if (Checker.isDefinitionAcrossSuspend(I, U))
- Spills.emplace_back(&I, U);
+ for (int Repeat = 0; Repeat < 4; ++Repeat) {
+ // See if there are materializable instructions across suspend points.
+ for (Instruction &I : instructions(F))
+ if (materializable(I))
+ for (User *U : I.users())
+ if (Checker.isDefinitionAcrossSuspend(I, U))
+ Spills.emplace_back(&I, U);
- // Rewrite materializable instructions to be materialized at the use point.
- DEBUG(dump("Materializations", Spills));
- rewriteMaterializableInstructions(Builder, Spills);
+ if (Spills.empty())
+ break;
+
+ // Rewrite materializable instructions to be materialized at the use point.
+ DEBUG(dump("Materializations", Spills));
+ rewriteMaterializableInstructions(Builder, Spills);
+ Spills.clear();
+ }
// Collect the spills for arguments and other not-materializable values.
- Spills.clear();
for (Argument &A : F.args())
for (User *U : A.users())
if (Checker.isDefinitionAcrossSuspend(A, U))
@@ -847,8 +852,6 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
if (I.getType()->isTokenTy())
report_fatal_error(
"token definition is separated from the use by a suspend point");
- assert(!materializable(I) &&
- "rewriteMaterializable did not do its job");
Spills.emplace_back(&I, U);
}
}
diff --git a/lib/Transforms/Coroutines/CoroSplit.cpp b/lib/Transforms/Coroutines/CoroSplit.cpp
index 12eb167898259..cd549e4be2826 100644
--- a/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -228,15 +228,7 @@ static Function *createClone(Function &F, Twine Suffix, coro::Shape &Shape,
SmallVector<ReturnInst *, 4> Returns;
- if (DISubprogram *SP = F.getSubprogram()) {
- // If we have debug info, add mapping for the metadata nodes that should not
- // be cloned by CloneFunctionInfo.
- auto &MD = VMap.MD();
- MD[SP->getUnit()].reset(SP->getUnit());
- MD[SP->getType()].reset(SP->getType());
- MD[SP->getFile()].reset(SP->getFile());
- }
- CloneFunctionInto(NewF, &F, VMap, /*ModuleLevelChanges=*/true, Returns);
+ CloneFunctionInto(NewF, &F, VMap, /*ModuleLevelChanges=*/false, Returns);
// Remove old returns.
for (ReturnInst *Return : Returns)
@@ -509,12 +501,87 @@ static void simplifySuspendPoints(coro::Shape &Shape) {
S.resize(N);
}
+static SmallPtrSet<BasicBlock *, 4> getCoroBeginPredBlocks(CoroBeginInst *CB) {
+ // Collect all blocks that we need to look for instructions to relocate.
+ SmallPtrSet<BasicBlock *, 4> RelocBlocks;
+ SmallVector<BasicBlock *, 4> Work;
+ Work.push_back(CB->getParent());
+
+ do {
+ BasicBlock *Current = Work.pop_back_val();
+ for (BasicBlock *BB : predecessors(Current))
+ if (RelocBlocks.count(BB) == 0) {
+ RelocBlocks.insert(BB);
+ Work.push_back(BB);
+ }
+ } while (!Work.empty());
+ return RelocBlocks;
+}
+
+static SmallPtrSet<Instruction *, 8>
+getNotRelocatableInstructions(CoroBeginInst *CoroBegin,
+ SmallPtrSetImpl<BasicBlock *> &RelocBlocks) {
+ SmallPtrSet<Instruction *, 8> DoNotRelocate;
+ // Collect all instructions that we should not relocate
+ SmallVector<Instruction *, 8> Work;
+
+ // Start with CoroBegin and terminators of all preceding blocks.
+ Work.push_back(CoroBegin);
+ BasicBlock *CoroBeginBB = CoroBegin->getParent();
+ for (BasicBlock *BB : RelocBlocks)
+ if (BB != CoroBeginBB)
+ Work.push_back(BB->getTerminator());
+
+ // For every instruction in the Work list, place its operands in DoNotRelocate
+ // set.
+ do {
+ Instruction *Current = Work.pop_back_val();
+ DoNotRelocate.insert(Current);
+ for (Value *U : Current->operands()) {
+ auto *I = dyn_cast<Instruction>(U);
+ if (!I)
+ continue;
+ if (isa<AllocaInst>(U))
+ continue;
+ if (DoNotRelocate.count(I) == 0) {
+ Work.push_back(I);
+ DoNotRelocate.insert(I);
+ }
+ }
+ } while (!Work.empty());
+ return DoNotRelocate;
+}
+
+static void relocateInstructionBefore(CoroBeginInst *CoroBegin, Function &F) {
+ // Analyze which non-alloca instructions are needed for allocation and
+ // relocate the rest to after coro.begin. We need to do it, since some of the
+ // targets of those instructions may be placed into coroutine frame memory
+ // for which becomes available after coro.begin intrinsic.
+
+ auto BlockSet = getCoroBeginPredBlocks(CoroBegin);
+ auto DoNotRelocateSet = getNotRelocatableInstructions(CoroBegin, BlockSet);
+
+ Instruction *InsertPt = CoroBegin->getNextNode();
+ BasicBlock &BB = F.getEntryBlock(); // TODO: Look at other blocks as well.
+ for (auto B = BB.begin(), E = BB.end(); B != E;) {
+ Instruction &I = *B++;
+ if (isa<AllocaInst>(&I))
+ continue;
+ if (&I == CoroBegin)
+ break;
+ if (DoNotRelocateSet.count(&I))
+ continue;
+ I.moveBefore(InsertPt);
+ }
+}
+
static void splitCoroutine(Function &F, CallGraph &CG, CallGraphSCC &SCC) {
coro::Shape Shape(F);
if (!Shape.CoroBegin)
return;
simplifySuspendPoints(Shape);
+ relocateInstructionBefore(Shape.CoroBegin, F);
buildCoroutineFrame(F, Shape);
replaceFrameSize(Shape);
@@ -660,6 +727,7 @@ struct CoroSplit : public CallGraphSCCPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
CallGraphSCCPass::getAnalysisUsage(AU);
}
+ StringRef getPassName() const override { return "Coroutine Splitting"; }
};
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-13 03:35:32 +0000