aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Coroutines
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Coroutines')
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroCleanup.cpp81
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroConditionalWrapper.cpp24
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroEarly.cpp79
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroElide.cpp125
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroFrame.cpp177
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroInternal.h47
-rw-r--r--llvm/lib/Transforms/Coroutines/CoroSplit.cpp377
-rw-r--r--llvm/lib/Transforms/Coroutines/Coroutines.cpp193
8 files changed, 310 insertions, 793 deletions
diff --git a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
index 67f8828e4c75..f7bbdcffd2ec 100644
--- a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
@@ -10,9 +10,9 @@
#include "CoroInternal.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
-#include "llvm/IR/LegacyPassManager.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/Scalar.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
using namespace llvm;
@@ -23,19 +23,10 @@ namespace {
struct Lowerer : coro::LowererBase {
IRBuilder<> Builder;
Lowerer(Module &M) : LowererBase(M), Builder(Context) {}
- bool lowerRemainingCoroIntrinsics(Function &F);
+ bool lower(Function &F);
};
}
-static void simplifyCFG(Function &F) {
- llvm::legacy::FunctionPassManager FPM(F.getParent());
- FPM.add(createCFGSimplificationPass());
-
- FPM.doInitialization();
- FPM.run(F);
- FPM.doFinalization();
-}
-
static void lowerSubFn(IRBuilder<> &Builder, CoroSubFnInst *SubFn) {
Builder.SetInsertPoint(SubFn);
Value *FrameRaw = SubFn->getFrame();
@@ -53,12 +44,10 @@ static void lowerSubFn(IRBuilder<> &Builder, CoroSubFnInst *SubFn) {
SubFn->replaceAllUsesWith(Load);
}
-bool Lowerer::lowerRemainingCoroIntrinsics(Function &F) {
+bool Lowerer::lower(Function &F) {
+ bool IsPrivateAndUnprocessed = F.isPresplitCoroutine() && F.hasLocalLinkage();
bool Changed = false;
- bool IsPrivateAndUnprocessed =
- F.hasFnAttribute(CORO_PRESPLIT_ATTR) && F.hasLocalLinkage();
-
for (Instruction &I : llvm::make_early_inc_range(instructions(F))) {
if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
switch (II->getIntrinsicID()) {
@@ -116,11 +105,6 @@ bool Lowerer::lowerRemainingCoroIntrinsics(Function &F) {
}
}
- if (Changed) {
- // After replacement were made we can cleanup the function body a little.
- simplifyCFG(F);
- }
-
return Changed;
}
@@ -132,50 +116,21 @@ static bool declaresCoroCleanupIntrinsics(const Module &M) {
"llvm.coro.async.resume"});
}
-PreservedAnalyses CoroCleanupPass::run(Function &F,
- FunctionAnalysisManager &AM) {
- auto &M = *F.getParent();
- if (!declaresCoroCleanupIntrinsics(M) ||
- !Lowerer(M).lowerRemainingCoroIntrinsics(F))
+PreservedAnalyses CoroCleanupPass::run(Module &M,
+ ModuleAnalysisManager &MAM) {
+ if (!declaresCoroCleanupIntrinsics(M))
return PreservedAnalyses::all();
- return PreservedAnalyses::none();
-}
-
-namespace {
-
-struct CoroCleanupLegacy : FunctionPass {
- static char ID; // Pass identification, replacement for typeid
+ FunctionAnalysisManager &FAM =
+ MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- CoroCleanupLegacy() : FunctionPass(ID) {
- initializeCoroCleanupLegacyPass(*PassRegistry::getPassRegistry());
- }
+ FunctionPassManager FPM;
+ FPM.addPass(SimplifyCFGPass());
- std::unique_ptr<Lowerer> L;
+ Lowerer L(M);
+ for (auto &F : M)
+ if (L.lower(F))
+ FPM.run(F, FAM);
- // This pass has work to do only if we find intrinsics we are going to lower
- // in the module.
- bool doInitialization(Module &M) override {
- if (declaresCoroCleanupIntrinsics(M))
- L = std::make_unique<Lowerer>(M);
- return false;
- }
-
- bool runOnFunction(Function &F) override {
- if (L)
- return L->lowerRemainingCoroIntrinsics(F);
- return false;
- }
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- if (!L)
- AU.setPreservesAll();
- }
- StringRef getPassName() const override { return "Coroutine Cleanup"; }
-};
+ return PreservedAnalyses::none();
}
-
-char CoroCleanupLegacy::ID = 0;
-INITIALIZE_PASS(CoroCleanupLegacy, "coro-cleanup",
- "Lower all coroutine related intrinsics", false, false)
-
-Pass *llvm::createCoroCleanupLegacyPass() { return new CoroCleanupLegacy(); }
diff --git a/llvm/lib/Transforms/Coroutines/CoroConditionalWrapper.cpp b/llvm/lib/Transforms/Coroutines/CoroConditionalWrapper.cpp
new file mode 100644
index 000000000000..3d26a43ceba7
--- /dev/null
+++ b/llvm/lib/Transforms/Coroutines/CoroConditionalWrapper.cpp
@@ -0,0 +1,24 @@
+//===- CoroConditionalWrapper.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
+#include "CoroInternal.h"
+#include "llvm/IR/Module.h"
+
+using namespace llvm;
+
+CoroConditionalWrapper::CoroConditionalWrapper(ModulePassManager &&PM)
+ : PM(std::move(PM)) {}
+
+PreservedAnalyses CoroConditionalWrapper::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ if (!coro::declaresAnyIntrinsic(M))
+ return PreservedAnalyses::all();
+
+ return PM.run(M, AM);
+}
diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index 1533e1805f17..dd7cb23f3f3d 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -8,10 +8,10 @@
#include "llvm/Transforms/Coroutines/CoroEarly.h"
#include "CoroInternal.h"
+#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Module.h"
-#include "llvm/Pass.h"
using namespace llvm;
@@ -35,7 +35,7 @@ public:
AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
/*isVarArg=*/false)
->getPointerTo()) {}
- bool lowerEarlyIntrinsics(Function &F);
+ void lowerEarlyIntrinsics(Function &F);
};
}
@@ -145,14 +145,16 @@ static void setCannotDuplicate(CoroIdInst *CoroId) {
CB->setCannotDuplicate();
}
-bool Lowerer::lowerEarlyIntrinsics(Function &F) {
- bool Changed = false;
+void Lowerer::lowerEarlyIntrinsics(Function &F) {
CoroIdInst *CoroId = nullptr;
SmallVector<CoroFreeInst *, 4> CoroFrees;
bool HasCoroSuspend = false;
for (Instruction &I : llvm::make_early_inc_range(instructions(F))) {
- if (auto *CB = dyn_cast<CallBase>(&I)) {
- switch (CB->getIntrinsicID()) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ continue;
+
+ switch (CB->getIntrinsicID()) {
default:
continue;
case Intrinsic::coro_free:
@@ -178,12 +180,9 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) {
case Intrinsic::coro_id:
if (auto *CII = cast<CoroIdInst>(&I)) {
if (CII->getInfo().isPreSplit()) {
- assert(F.hasFnAttribute(CORO_PRESPLIT_ATTR) &&
- F.getFnAttribute(CORO_PRESPLIT_ATTR).getValueAsString() ==
- UNPREPARED_FOR_SPLIT &&
+ assert(F.isPresplitCoroutine() &&
"The frontend uses Swtich-Resumed ABI should emit "
- "\"coroutine.presplit\" attribute with value \"0\" for the "
- "coroutine.");
+ "\"coroutine.presplit\" attribute for the coroutine.");
setCannotDuplicate(CII);
CII->setCoroutineSelf();
CoroId = cast<CoroIdInst>(&I);
@@ -193,9 +192,7 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) {
case Intrinsic::coro_id_retcon:
case Intrinsic::coro_id_retcon_once:
case Intrinsic::coro_id_async:
- // TODO: Remove the line once we support it in the corresponding
- // frontend.
- F.addFnAttr(CORO_PRESPLIT_ATTR, PREPARED_FOR_SPLIT);
+ F.setPresplitCoroutine();
break;
case Intrinsic::coro_resume:
lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
@@ -209,16 +206,16 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) {
case Intrinsic::coro_done:
lowerCoroDone(cast<IntrinsicInst>(&I));
break;
- }
- Changed = true;
}
}
+
// Make sure that all CoroFree reference the coro.id intrinsic.
// Token type is not exposed through coroutine C/C++ builtins to plain C, so
// we allow specifying none and fixing it up here.
if (CoroId)
for (CoroFreeInst *CF : CoroFrees)
CF->setArgOperand(0, CoroId);
+
// Coroutine suspention could potentially lead to any argument modified
// outside of the function, hence arguments should not have noalias
// attributes.
@@ -226,7 +223,6 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) {
for (Argument &A : F.args())
if (A.hasNoAliasAttr())
A.removeAttr(Attribute::NoAlias);
- return Changed;
}
static bool declaresCoroEarlyIntrinsics(const Module &M) {
@@ -238,52 +234,15 @@ static bool declaresCoroEarlyIntrinsics(const Module &M) {
"llvm.coro.suspend"});
}
-PreservedAnalyses CoroEarlyPass::run(Function &F, FunctionAnalysisManager &) {
- Module &M = *F.getParent();
- if (!declaresCoroEarlyIntrinsics(M) || !Lowerer(M).lowerEarlyIntrinsics(F))
+PreservedAnalyses CoroEarlyPass::run(Module &M, ModuleAnalysisManager &) {
+ if (!declaresCoroEarlyIntrinsics(M))
return PreservedAnalyses::all();
+ Lowerer L(M);
+ for (auto &F : M)
+ L.lowerEarlyIntrinsics(F);
+
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
return PA;
}
-
-namespace {
-
-struct CoroEarlyLegacy : public FunctionPass {
- static char ID; // Pass identification, replacement for typeid.
- CoroEarlyLegacy() : FunctionPass(ID) {
- initializeCoroEarlyLegacyPass(*PassRegistry::getPassRegistry());
- }
-
- std::unique_ptr<Lowerer> L;
-
- // This pass has work to do only if we find intrinsics we are going to lower
- // in the module.
- bool doInitialization(Module &M) override {
- if (declaresCoroEarlyIntrinsics(M))
- L = std::make_unique<Lowerer>(M);
- return false;
- }
-
- bool runOnFunction(Function &F) override {
- if (!L)
- return false;
-
- return L->lowerEarlyIntrinsics(F);
- }
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
- }
- StringRef getPassName() const override {
- return "Lower early coroutine intrinsics";
- }
-};
-}
-
-char CoroEarlyLegacy::ID = 0;
-INITIALIZE_PASS(CoroEarlyLegacy, "coro-early",
- "Lower early coroutine intrinsics", false, false)
-
-Pass *llvm::createCoroEarlyLegacyPass() { return new CoroEarlyLegacy(); }
diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
index 84bebb7bf42d..6f78fc8db311 100644
--- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
@@ -14,8 +14,6 @@
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/InstIterator.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
@@ -103,21 +101,12 @@ static void removeTailCallAttribute(AllocaInst *Frame, AAResults &AA) {
// Given a resume function @f.resume(%f.frame* %frame), returns the size
// and expected alignment of %f.frame type.
-static std::pair<uint64_t, Align> getFrameLayout(Function *Resume) {
- // Prefer to pull information from the function attributes.
+static Optional<std::pair<uint64_t, Align>> getFrameLayout(Function *Resume) {
+ // Pull information from the function attributes.
auto Size = Resume->getParamDereferenceableBytes(0);
- auto Align = Resume->getParamAlign(0);
-
- // If those aren't given, extract them from the type.
- if (Size == 0 || !Align) {
- auto *FrameTy = Resume->arg_begin()->getType()->getPointerElementType();
-
- const DataLayout &DL = Resume->getParent()->getDataLayout();
- if (!Size) Size = DL.getTypeAllocSize(FrameTy);
- if (!Align) Align = DL.getABITypeAlign(FrameTy);
- }
-
- return std::make_pair(Size, *Align);
+ if (!Size)
+ return None;
+ return std::make_pair(Size, Resume->getParamAlign(0).valueOrOne());
}
// Finds first non alloca instruction in the entry block of a function.
@@ -347,56 +336,37 @@ bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA,
assert(Resumers && "PostSplit coro.id Info argument must refer to an array"
"of coroutine subfunctions");
auto *ResumeAddrConstant =
- ConstantExpr::getExtractValue(Resumers, CoroSubFnInst::ResumeIndex);
+ Resumers->getAggregateElement(CoroSubFnInst::ResumeIndex);
replaceWithConstant(ResumeAddrConstant, ResumeAddr);
bool ShouldElide = shouldElide(CoroId->getFunction(), DT);
- auto *DestroyAddrConstant = ConstantExpr::getExtractValue(
- Resumers,
+ auto *DestroyAddrConstant = Resumers->getAggregateElement(
ShouldElide ? CoroSubFnInst::CleanupIndex : CoroSubFnInst::DestroyIndex);
for (auto &It : DestroyAddr)
replaceWithConstant(DestroyAddrConstant, It.second);
if (ShouldElide) {
- auto FrameSizeAndAlign = getFrameLayout(cast<Function>(ResumeAddrConstant));
- elideHeapAllocations(CoroId->getFunction(), FrameSizeAndAlign.first,
- FrameSizeAndAlign.second, AA);
- coro::replaceCoroFree(CoroId, /*Elide=*/true);
- NumOfCoroElided++;
+ if (auto FrameSizeAndAlign =
+ getFrameLayout(cast<Function>(ResumeAddrConstant))) {
+ elideHeapAllocations(CoroId->getFunction(), FrameSizeAndAlign->first,
+ FrameSizeAndAlign->second, AA);
+ coro::replaceCoroFree(CoroId, /*Elide=*/true);
+ NumOfCoroElided++;
#ifndef NDEBUG
- if (!CoroElideInfoOutputFilename.empty())
- *getOrCreateLogFile()
- << "Elide " << CoroId->getCoroutine()->getName() << " in "
- << CoroId->getFunction()->getName() << "\n";
+ if (!CoroElideInfoOutputFilename.empty())
+ *getOrCreateLogFile()
+ << "Elide " << CoroId->getCoroutine()->getName() << " in "
+ << CoroId->getFunction()->getName() << "\n";
#endif
+ }
}
return true;
}
-// See if there are any coro.subfn.addr instructions referring to coro.devirt
-// trigger, if so, replace them with a direct call to devirt trigger function.
-static bool replaceDevirtTrigger(Function &F) {
- SmallVector<CoroSubFnInst *, 1> DevirtAddr;
- for (auto &I : instructions(F))
- if (auto *SubFn = dyn_cast<CoroSubFnInst>(&I))
- if (SubFn->getIndex() == CoroSubFnInst::RestartTrigger)
- DevirtAddr.push_back(SubFn);
-
- if (DevirtAddr.empty())
- return false;
-
- Module &M = *F.getParent();
- Function *DevirtFn = M.getFunction(CORO_DEVIRT_TRIGGER_FN);
- assert(DevirtFn && "coro.devirt.fn not found");
- replaceWithConstant(DevirtFn, DevirtAddr);
-
- return true;
-}
-
static bool declaresCoroElideIntrinsics(Module &M) {
return coro::declaresIntrinsics(M, {"llvm.coro.id", "llvm.coro.id.async"});
}
@@ -422,62 +392,3 @@ PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) {
return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
}
-
-namespace {
-struct CoroElideLegacy : FunctionPass {
- static char ID;
- CoroElideLegacy() : FunctionPass(ID) {
- initializeCoroElideLegacyPass(*PassRegistry::getPassRegistry());
- }
-
- std::unique_ptr<Lowerer> L;
-
- bool doInitialization(Module &M) override {
- if (declaresCoroElideIntrinsics(M))
- L = std::make_unique<Lowerer>(M);
- return false;
- }
-
- bool runOnFunction(Function &F) override {
- if (!L)
- return false;
-
- bool Changed = false;
-
- if (F.hasFnAttribute(CORO_PRESPLIT_ATTR))
- Changed = replaceDevirtTrigger(F);
-
- L->CoroIds.clear();
- L->collectPostSplitCoroIds(&F);
- // If we did not find any coro.id, there is nothing to do.
- if (L->CoroIds.empty())
- return Changed;
-
- AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
- DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-
- for (auto *CII : L->CoroIds)
- Changed |= L->processCoroId(CII, AA, DT);
-
- return Changed;
- }
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<AAResultsWrapperPass>();
- AU.addRequired<DominatorTreeWrapperPass>();
- }
- StringRef getPassName() const override { return "Coroutine Elision"; }
-};
-}
-
-char CoroElideLegacy::ID = 0;
-INITIALIZE_PASS_BEGIN(
- CoroElideLegacy, "coro-elide",
- "Coroutine frame allocation elision and indirect calls replacement", false,
- false)
-INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
-INITIALIZE_PASS_END(
- CoroElideLegacy, "coro-elide",
- "Coroutine frame allocation elision and indirect calls replacement", false,
- false)
-
-Pass *llvm::createCoroElideLegacyPass() { return new CoroElideLegacy(); }
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 9c16d3750998..d09607bb1c4c 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -27,7 +27,7 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
-#include "llvm/Support/CommandLine.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/OptimizedStructLayout.h"
@@ -44,13 +44,6 @@ using namespace llvm;
// "coro-frame", which results in leaner debug spew.
#define DEBUG_TYPE "coro-suspend-crossing"
-static cl::opt<bool> EnableReuseStorageInFrame(
- "reuse-storage-in-coroutine-frame", cl::Hidden,
- cl::desc(
- "Enable the optimization which would reuse the storage in the coroutine \
- frame for allocas whose liferanges are not overlapped, for testing purposes"),
- llvm::cl::init(false));
-
enum { SmallVectorThreshold = 32 };
// Provides two way mapping between the blocks and numbers.
@@ -347,15 +340,26 @@ struct FrameDataInfo {
FieldIndexMap[V] = Index;
}
- uint64_t getAlign(Value *V) const {
+ Align getAlign(Value *V) const {
auto Iter = FieldAlignMap.find(V);
assert(Iter != FieldAlignMap.end());
return Iter->second;
}
- void setAlign(Value *V, uint64_t Align) {
+ void setAlign(Value *V, Align AL) {
assert(FieldAlignMap.count(V) == 0);
- FieldAlignMap.insert({V, Align});
+ FieldAlignMap.insert({V, AL});
+ }
+
+ uint64_t getDynamicAlign(Value *V) const {
+ auto Iter = FieldDynamicAlignMap.find(V);
+ assert(Iter != FieldDynamicAlignMap.end());
+ return Iter->second;
+ }
+
+ void setDynamicAlign(Value *V, uint64_t Align) {
+ assert(FieldDynamicAlignMap.count(V) == 0);
+ FieldDynamicAlignMap.insert({V, Align});
}
uint64_t getOffset(Value *V) const {
@@ -382,7 +386,8 @@ private:
DenseMap<Value *, uint32_t> FieldIndexMap;
// Map from values to their alignment on the frame. They would be set after
// the frame is built.
- DenseMap<Value *, uint64_t> FieldAlignMap;
+ DenseMap<Value *, Align> FieldAlignMap;
+ DenseMap<Value *, uint64_t> FieldDynamicAlignMap;
// Map from values to their offset on the frame. They would be set after
// the frame is built.
DenseMap<Value *, uint64_t> FieldOffsetMap;
@@ -423,6 +428,7 @@ private:
FieldIDType LayoutFieldIndex;
Align Alignment;
Align TyAlignment;
+ uint64_t DynamicAlignBuffer;
};
const DataLayout &DL;
@@ -489,7 +495,7 @@ public:
coro::Shape &Shape);
/// Add a field to this structure.
- LLVM_NODISCARD FieldIDType addField(Type *Ty, MaybeAlign FieldAlignment,
+ LLVM_NODISCARD FieldIDType addField(Type *Ty, MaybeAlign MaybeFieldAlignment,
bool IsHeader = false,
bool IsSpillOfValue = false) {
assert(!IsFinished && "adding fields to a finished builder");
@@ -508,13 +514,21 @@ public:
// to remember the type alignment anyway to build the type.
// If we are spilling values we don't need to worry about ABI alignment
// concerns.
- auto ABIAlign = DL.getABITypeAlign(Ty);
- Align TyAlignment =
- (IsSpillOfValue && MaxFrameAlignment)
- ? (*MaxFrameAlignment < ABIAlign ? *MaxFrameAlignment : ABIAlign)
- : ABIAlign;
- if (!FieldAlignment) {
- FieldAlignment = TyAlignment;
+ Align ABIAlign = DL.getABITypeAlign(Ty);
+ Align TyAlignment = ABIAlign;
+ if (IsSpillOfValue && MaxFrameAlignment && *MaxFrameAlignment < ABIAlign)
+ TyAlignment = *MaxFrameAlignment;
+ Align FieldAlignment = MaybeFieldAlignment.value_or(TyAlignment);
+
+ // The field alignment could be bigger than the max frame case, in that case
+ // we request additional storage to be able to dynamically align the
+ // pointer.
+ uint64_t DynamicAlignBuffer = 0;
+ if (MaxFrameAlignment && (FieldAlignment > *MaxFrameAlignment)) {
+ DynamicAlignBuffer =
+ offsetToAlignment(MaxFrameAlignment->value(), FieldAlignment);
+ FieldAlignment = *MaxFrameAlignment;
+ FieldSize = FieldSize + DynamicAlignBuffer;
}
// Lay out header fields immediately.
@@ -523,12 +537,13 @@ public:
Offset = alignTo(StructSize, FieldAlignment);
StructSize = Offset + FieldSize;
- // Everything else has a flexible offset.
+ // Everything else has a flexible offset.
} else {
Offset = OptimizedStructLayoutField::FlexibleOffset;
}
- Fields.push_back({FieldSize, Offset, Ty, 0, *FieldAlignment, TyAlignment});
+ Fields.push_back({FieldSize, Offset, Ty, 0, FieldAlignment, TyAlignment,
+ DynamicAlignBuffer});
return Fields.size() - 1;
}
@@ -561,7 +576,12 @@ void FrameDataInfo::updateLayoutIndex(FrameTypeBuilder &B) {
auto Updater = [&](Value *I) {
auto Field = B.getLayoutField(getFieldIndex(I));
setFieldIndex(I, Field.LayoutFieldIndex);
- setAlign(I, Field.Alignment.value());
+ setAlign(I, Field.Alignment);
+ uint64_t dynamicAlign =
+ Field.DynamicAlignBuffer
+ ? Field.DynamicAlignBuffer + Field.Alignment.value()
+ : 0;
+ setDynamicAlign(I, dynamicAlign);
setOffset(I, Field.Offset);
};
LayoutIndexUpdateStarted = true;
@@ -588,7 +608,7 @@ void FrameTypeBuilder::addFieldForAllocas(const Function &F,
}
});
- if (!Shape.OptimizeFrame && !EnableReuseStorageInFrame) {
+ if (!Shape.OptimizeFrame) {
for (const auto &A : FrameData.Allocas) {
AllocaInst *Alloca = A.Alloca;
NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
@@ -755,6 +775,10 @@ void FrameTypeBuilder::finish(StructType *Ty) {
F.LayoutFieldIndex = FieldTypes.size();
FieldTypes.push_back(F.Ty);
+ if (F.DynamicAlignBuffer) {
+ FieldTypes.push_back(
+ ArrayType::get(Type::getInt8Ty(Context), F.DynamicAlignBuffer));
+ }
LastOffset = Offset + F.Size;
}
@@ -807,9 +831,10 @@ static StringRef solveTypeName(Type *Ty) {
return "__floating_type_";
}
- if (Ty->isPointerTy()) {
- auto *PtrTy = cast<PointerType>(Ty);
- Type *PointeeTy = PtrTy->getPointerElementType();
+ if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
+ if (PtrTy->isOpaque())
+ return "PointerType";
+ Type *PointeeTy = PtrTy->getNonOpaquePointerElementType();
auto Name = solveTypeName(PointeeTy);
if (Name == "UnknownType")
return "PointerType";
@@ -826,10 +851,9 @@ static StringRef solveTypeName(Type *Ty) {
auto Name = Ty->getStructName();
SmallString<16> Buffer(Name);
- for_each(Buffer, [](auto &Iter) {
+ for (auto &Iter : Buffer)
if (Iter == '.' || Iter == ':')
Iter = '_';
- });
auto *MDName = MDString::get(Ty->getContext(), Buffer.str());
return MDName->getString();
}
@@ -1012,7 +1036,7 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
auto Index = FrameData.getFieldIndex(V);
OffsetCache.insert(
- {Index, {FrameData.getAlign(V), FrameData.getOffset(V)}});
+ {Index, {FrameData.getAlign(V).value(), FrameData.getOffset(V)}});
}
DenseMap<Type *, DIType *> DITypeCache;
@@ -1078,7 +1102,7 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
DBuilder.createExpression(), DILoc,
- Shape.FramePtr->getNextNode());
+ Shape.getInsertPtAfterFramePtr());
}
// Build a struct that will keep state for an active coroutine.
@@ -1367,7 +1391,7 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
bool getShouldLiveOnFrame() const {
if (!ShouldLiveOnFrame)
ShouldLiveOnFrame = computeShouldLiveOnFrame();
- return ShouldLiveOnFrame.getValue();
+ return *ShouldLiveOnFrame;
}
bool getMayWriteBeforeCoroBegin() const { return MayWriteBeforeCoroBegin; }
@@ -1455,7 +1479,7 @@ private:
auto Itr = AliasOffetMap.find(&I);
if (Itr == AliasOffetMap.end()) {
AliasOffetMap[&I] = Offset;
- } else if (Itr->second.hasValue() && Itr->second.getValue() != Offset) {
+ } else if (Itr->second && *Itr->second != Offset) {
// If we have seen two different possible values for this alias, we set
// it to empty.
AliasOffetMap[&I].reset();
@@ -1517,13 +1541,12 @@ static void createFramePtr(coro::Shape &Shape) {
// whatever
//
//
-static Instruction *insertSpills(const FrameDataInfo &FrameData,
- coro::Shape &Shape) {
+static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
auto *CB = Shape.CoroBegin;
LLVMContext &C = CB->getContext();
IRBuilder<> Builder(C);
StructType *FrameTy = Shape.FrameTy;
- Instruction *FramePtr = Shape.FramePtr;
+ Value *FramePtr = Shape.FramePtr;
DominatorTree DT(*CB->getFunction());
SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache;
@@ -1550,7 +1573,18 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
auto GEP = cast<GetElementPtrInst>(
Builder.CreateInBoundsGEP(FrameTy, FramePtr, Indices));
- if (isa<AllocaInst>(Orig)) {
+ if (auto *AI = dyn_cast<AllocaInst>(Orig)) {
+ if (FrameData.getDynamicAlign(Orig) != 0) {
+ assert(FrameData.getDynamicAlign(Orig) == AI->getAlign().value());
+ auto *M = AI->getModule();
+ auto *IntPtrTy = M->getDataLayout().getIntPtrType(AI->getType());
+ auto *PtrValue = Builder.CreatePtrToInt(GEP, IntPtrTy);
+ auto *AlignMask =
+ ConstantInt::get(IntPtrTy, AI->getAlign().value() - 1);
+ PtrValue = Builder.CreateAdd(PtrValue, AlignMask);
+ PtrValue = Builder.CreateAnd(PtrValue, Builder.CreateNot(AlignMask));
+ return Builder.CreateIntToPtr(PtrValue, AI->getType());
+ }
// If the type of GEP is not equal to the type of AllocaInst, it implies
// that the AllocaInst may be reused in the Frame slot of other
// AllocaInst. So We cast GEP to the AllocaInst here to re-use
@@ -1571,20 +1605,19 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
// Create a store instruction storing the value into the
// coroutine frame.
Instruction *InsertPt = nullptr;
- bool NeedToCopyArgPtrValue = false;
+ Type *ByValTy = nullptr;
if (auto *Arg = dyn_cast<Argument>(Def)) {
// For arguments, we will place the store instruction right after
// the coroutine frame pointer instruction, i.e. bitcast of
// coro.begin from i8* to %f.frame*.
- InsertPt = FramePtr->getNextNode();
+ InsertPt = Shape.getInsertPtAfterFramePtr();
// If we're spilling an Argument, make sure we clear 'nocapture'
// from the coroutine function.
Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::NoCapture);
if (Arg->hasByValAttr())
- NeedToCopyArgPtrValue = true;
-
+ ByValTy = Arg->getParamByValType();
} else if (auto *CSI = dyn_cast<AnyCoroSuspendInst>(Def)) {
// Don't spill immediately after a suspend; splitting assumes
// that the suspend will be followed by a branch.
@@ -1594,7 +1627,7 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
if (!DT.dominates(CB, I)) {
// If it is not dominated by CoroBegin, then spill should be
// inserted immediately after CoroFrame is computed.
- InsertPt = FramePtr->getNextNode();
+ InsertPt = Shape.getInsertPtAfterFramePtr();
} else if (auto *II = dyn_cast<InvokeInst>(I)) {
// If we are spilling the result of the invoke instruction, split
// the normal edge and insert the spill in the new block.
@@ -1619,11 +1652,10 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
Builder.SetInsertPoint(InsertPt);
auto *G = Builder.CreateConstInBoundsGEP2_32(
FrameTy, FramePtr, 0, Index, Def->getName() + Twine(".spill.addr"));
- if (NeedToCopyArgPtrValue) {
+ if (ByValTy) {
// For byval arguments, we need to store the pointed value in the frame,
// instead of the pointer itself.
- auto *Value =
- Builder.CreateLoad(Def->getType()->getPointerElementType(), Def);
+ auto *Value = Builder.CreateLoad(ByValTy, Def);
Builder.CreateAlignedStore(Value, G, SpillAlignment);
} else {
Builder.CreateAlignedStore(Def, G, SpillAlignment);
@@ -1641,7 +1673,7 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
auto *GEP = GetFramePointer(E.first);
GEP->setName(E.first->getName() + Twine(".reload.addr"));
- if (NeedToCopyArgPtrValue)
+ if (ByValTy)
CurrentReload = GEP;
else
CurrentReload = Builder.CreateAlignedLoad(
@@ -1664,6 +1696,12 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
}
}
+ // Salvage debug info on any dbg.addr that we see. We do not insert them
+ // into each block where we have a use though.
+ if (auto *DI = dyn_cast<DbgAddrIntrinsic>(U)) {
+ coro::salvageDebugInfo(DbgPtrAllocaCache, DI, Shape.OptimizeFrame);
+ }
+
// If we have a single edge PHINode, remove it and replace it with a
// reload from the coroutine frame. (We already took care of multi edge
// PHINodes by rewriting them in the rewritePHIs function).
@@ -1682,10 +1720,10 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
}
}
- BasicBlock *FramePtrBB = FramePtr->getParent();
+ BasicBlock *FramePtrBB = Shape.getInsertPtAfterFramePtr()->getParent();
- auto SpillBlock =
- FramePtrBB->splitBasicBlock(FramePtr->getNextNode(), "AllocaSpillBB");
+ auto SpillBlock = FramePtrBB->splitBasicBlock(
+ Shape.getInsertPtAfterFramePtr(), "AllocaSpillBB");
SpillBlock->splitBasicBlock(&SpillBlock->front(), "PostSpill");
Shape.AllocaSpillBlock = SpillBlock;
@@ -1704,7 +1742,7 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
Alloca->replaceAllUsesWith(G);
Alloca->eraseFromParent();
}
- return FramePtr;
+ return;
}
// If we found any alloca, replace all of their remaining uses with GEP
@@ -1735,7 +1773,7 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
for (Instruction *I : UsersToUpdate)
I->replaceUsesOfWith(Alloca, G);
}
- Builder.SetInsertPoint(FramePtr->getNextNode());
+ Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
for (const auto &A : FrameData.Allocas) {
AllocaInst *Alloca = A.Alloca;
if (A.MayWriteBeforeCoroBegin) {
@@ -1755,16 +1793,16 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
auto *FramePtr = GetFramePointer(Alloca);
auto *FramePtrRaw =
Builder.CreateBitCast(FramePtr, Type::getInt8PtrTy(C));
- auto *AliasPtr = Builder.CreateGEP(
- Type::getInt8Ty(C), FramePtrRaw,
- ConstantInt::get(Type::getInt64Ty(C), Alias.second.getValue()));
+ auto &Value = *Alias.second;
+ auto ITy = IntegerType::get(C, Value.getBitWidth());
+ auto *AliasPtr = Builder.CreateGEP(Type::getInt8Ty(C), FramePtrRaw,
+ ConstantInt::get(ITy, Value));
auto *AliasPtrTyped =
Builder.CreateBitCast(AliasPtr, Alias.first->getType());
Alias.first->replaceUsesWithIf(
AliasPtrTyped, [&](Use &U) { return DT.dominates(CB, U); });
}
}
- return FramePtr;
}
// Moves the values in the PHIs in SuccBB that correspong to PredBB into a new
@@ -2130,7 +2168,7 @@ static void lowerLocalAllocas(ArrayRef<CoroAllocaAllocInst*> LocalAllocas,
// Allocate memory.
auto Alloca = Builder.CreateAlloca(Builder.getInt8Ty(), AI->getSize());
- Alloca->setAlignment(Align(AI->getAlignment()));
+ Alloca->setAlignment(AI->getAlignment());
for (auto U : AI->users()) {
// Replace gets with the allocation.
@@ -2279,7 +2317,10 @@ static void eliminateSwiftErrorArgument(Function &F, Argument &Arg,
IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
auto ArgTy = cast<PointerType>(Arg.getType());
- auto ValueTy = ArgTy->getPointerElementType();
+ // swifterror arguments are required to have pointer-to-pointer type,
+ // so create a pointer-typed alloca with opaque pointers.
+ auto ValueTy = ArgTy->isOpaque() ? PointerType::getUnqual(F.getContext())
+ : ArgTy->getNonOpaquePointerElementType();
// Reduce to the alloca case:
@@ -2520,6 +2561,7 @@ void coro::salvageDebugInfo(
bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
Value *Storage = DVI->getVariableLocationOp(0);
Value *OriginalStorage = Storage;
+
while (auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
if (auto *LdInst = dyn_cast<LoadInst>(Inst)) {
Storage = LdInst->getOperand(0);
@@ -2559,7 +2601,7 @@ void coro::salvageDebugInfo(
//
// Avoid to create the alloca would be eliminated by optimization
// passes and the corresponding dbg.declares would be invalid.
- if (!OptimizeFrame && !EnableReuseStorageInFrame)
+ if (!OptimizeFrame)
if (auto *Arg = dyn_cast<llvm::Argument>(Storage)) {
auto &Cached = DbgPtrAllocaCache[Storage];
if (!Cached) {
@@ -2575,14 +2617,15 @@ void coro::salvageDebugInfo(
// expression, we need to add a DW_OP_deref at the *start* of the
// expression to first load the contents of the alloca before
// adjusting it with the expression.
- if (Expr && Expr->isComplex())
- Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
+ Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
}
DVI->replaceVariableLocationOp(OriginalStorage, Storage);
DVI->setExpression(Expr);
- /// It makes no sense to move the dbg.value intrinsic.
- if (!isa<DbgValueInst>(DVI)) {
+ // We only hoist dbg.declare today since it doesn't make sense to hoist
+ // dbg.value or dbg.addr since they do not have the same function wide
+ // guarantees that dbg.declare does.
+ if (!isa<DbgValueInst>(DVI) && !isa<DbgAddrIntrinsic>(DVI)) {
if (auto *II = dyn_cast<InvokeInst>(Storage))
DVI->moveBefore(II->getNormalDest()->getFirstNonPHI());
else if (auto *CBI = dyn_cast<CallBrInst>(Storage))
@@ -2661,13 +2704,6 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
for (User *U : I.users())
if (Checker.isDefinitionAcrossSuspend(I, U))
Spills[&I].push_back(cast<Instruction>(U));
-
- // Manually add dbg.value metadata uses of I.
- SmallVector<DbgValueInst *, 16> DVIs;
- findDbgValues(DVIs, &I);
- for (auto *DVI : DVIs)
- if (Checker.isDefinitionAcrossSuspend(I, DVI))
- Spills[&I].push_back(DVI);
}
if (Spills.empty())
@@ -2754,10 +2790,9 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
auto *V = Iter.first;
SmallVector<DbgValueInst *, 16> DVIs;
findDbgValues(DVIs, V);
- llvm::for_each(DVIs, [&](DbgValueInst *DVI) {
+ for (DbgValueInst *DVI : DVIs)
if (Checker.isDefinitionAcrossSuspend(*V, DVI))
FrameData.Spills[V].push_back(DVI);
- });
}
LLVM_DEBUG(dumpSpills("Spills", FrameData.Spills));
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index 9a17068df3a9..5557370c82ba 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -13,7 +13,6 @@
#include "CoroInstr.h"
#include "llvm/IR/IRBuilder.h"
-#include "llvm/Transforms/Coroutines.h"
namespace llvm {
@@ -21,40 +20,13 @@ class CallGraph;
class CallGraphSCC;
class PassRegistry;
-void initializeCoroEarlyLegacyPass(PassRegistry &);
-void initializeCoroSplitLegacyPass(PassRegistry &);
-void initializeCoroElideLegacyPass(PassRegistry &);
-void initializeCoroCleanupLegacyPass(PassRegistry &);
-
-// CoroEarly pass marks every function that has coro.begin with a string
-// attribute "coroutine.presplit"="0". CoroSplit pass processes the coroutine
-// twice. First, it lets it go through complete IPO optimization pipeline as a
-// single function. It forces restart of the pipeline by inserting an indirect
-// call to an empty function "coro.devirt.trigger" which is devirtualized by
-// CoroElide pass that triggers a restart of the pipeline by CGPassManager.
-// When CoroSplit pass sees the same coroutine the second time, it splits it up,
-// adds coroutine subfunctions to the SCC to be processed by IPO pipeline.
-// Async lowering similarily triggers a restart of the pipeline after it has
-// split the coroutine.
-//
-// FIXME: Refactor these attributes as LLVM attributes instead of string
-// attributes since these attributes are already used outside LLVM's
-// coroutine module.
-// FIXME: Remove these values once we remove the Legacy PM.
-#define CORO_PRESPLIT_ATTR "coroutine.presplit"
-#define UNPREPARED_FOR_SPLIT "0"
-#define PREPARED_FOR_SPLIT "1"
-#define ASYNC_RESTART_AFTER_SPLIT "2"
-
-#define CORO_DEVIRT_TRIGGER_FN "coro.devirt.trigger"
-
namespace coro {
+bool declaresAnyIntrinsic(const Module &M);
bool declaresIntrinsics(const Module &M,
const std::initializer_list<StringRef>);
void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
-void updateCallGraph(Function &Caller, ArrayRef<Function *> Funcs,
- CallGraph &CG, CallGraphSCC &SCC);
+
/// Recover a dbg.declare prepared by the frontend and emit an alloca
/// holding a pointer to the coroutine frame.
void salvageDebugInfo(
@@ -128,7 +100,7 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
StructType *FrameTy;
Align FrameAlign;
uint64_t FrameSize;
- Instruction *FramePtr;
+ Value *FramePtr;
BasicBlock *AllocaSpillBlock;
/// This would only be true if optimization are enabled.
@@ -210,10 +182,9 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
FunctionType *getResumeFunctionType() const {
switch (ABI) {
- case coro::ABI::Switch: {
- auto *FnPtrTy = getSwitchResumePointerType();
- return cast<FunctionType>(FnPtrTy->getPointerElementType());
- }
+ case coro::ABI::Switch:
+ return FunctionType::get(Type::getVoidTy(FrameTy->getContext()),
+ FrameTy->getPointerTo(), /*IsVarArg*/false);
case coro::ABI::Retcon:
case coro::ABI::RetconOnce:
return RetconLowering.ResumePrototype->getFunctionType();
@@ -267,6 +238,12 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
return nullptr;
}
+ Instruction *getInsertPtAfterFramePtr() const {
+ if (auto *I = dyn_cast<Instruction>(FramePtr))
+ return I->getNextNode();
+ return &cast<Argument>(FramePtr)->getParent()->getEntryBlock().front();
+ }
+
/// Allocate memory according to the rules of the active lowering.
///
/// \param CG - if non-null, will be updated for the new call
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index b5129809c6a6..ead552d9be4e 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -22,15 +22,17 @@
#include "CoroInstr.h"
#include "CoroInternal.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/PriorityWorklist.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CallGraph.h"
-#include "llvm/Analysis/CallGraphSCCPass.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
@@ -50,13 +52,10 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/PrettyStackTrace.h"
@@ -869,11 +868,16 @@ void CoroCloner::create() {
OrigF.getParent()->end(), ActiveSuspend);
}
- // Replace all args with undefs. The buildCoroutineFrame algorithm already
- // rewritten access to the args that occurs after suspend points with loads
- // and stores to/from the coroutine frame.
- for (Argument &A : OrigF.args())
- VMap[&A] = UndefValue::get(A.getType());
+ // Replace all args with dummy instructions. If an argument is the old frame
+ // pointer, the dummy will be replaced by the new frame pointer once it is
+ // computed below. Uses of all other arguments should have already been
+ // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
+ // frame.
+ SmallVector<Instruction *> DummyArgs;
+ for (Argument &A : OrigF.args()) {
+ DummyArgs.push_back(new FreezeInst(UndefValue::get(A.getType())));
+ VMap[&A] = DummyArgs.back();
+ }
SmallVector<ReturnInst *, 4> Returns;
@@ -923,6 +927,12 @@ void CoroCloner::create() {
NewF->setVisibility(savedVisibility);
NewF->setUnnamedAddr(savedUnnamedAddr);
NewF->setDLLStorageClass(savedDLLStorageClass);
+ // The function sanitizer metadata needs to match the signature of the
+ // function it is being attached to. However this does not hold for split
+ // functions here. Thus remove the metadata for split functions.
+ if (Shape.ABI == coro::ABI::Switch &&
+ NewF->hasMetadata(LLVMContext::MD_func_sanitize))
+ NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
// Replace the attributes of the new function:
auto OrigAttrs = NewF->getAttributes();
@@ -932,7 +942,8 @@ void CoroCloner::create() {
case coro::ABI::Switch:
// Bootstrap attributes by copying function attributes from the
// original function. This should include optimization settings and so on.
- NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
+ NewAttrs = NewAttrs.addFnAttributes(
+ Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
addFramePointerAttrs(NewAttrs, Context, 0,
Shape.FrameSize, Shape.FrameAlign);
@@ -1013,7 +1024,15 @@ void CoroCloner::create() {
auto *NewVFrame = Builder.CreateBitCast(
NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
- OldVFrame->replaceAllUsesWith(NewVFrame);
+ if (OldVFrame != NewVFrame)
+ OldVFrame->replaceAllUsesWith(NewVFrame);
+
+ // All uses of the arguments should have been resolved by this point,
+ // so we can safely remove the dummy values.
+ for (Instruction *DummyArg : DummyArgs) {
+ DummyArg->replaceAllUsesWith(UndefValue::get(DummyArg->getType()));
+ DummyArg->deleteValue();
+ }
switch (Shape.ABI) {
case coro::ABI::Switch:
@@ -1063,13 +1082,6 @@ static Function *createClone(Function &F, const Twine &Suffix,
return Cloner.getFunction();
}
-/// Remove calls to llvm.coro.end in the original function.
-static void removeCoroEnds(const coro::Shape &Shape, CallGraph *CG) {
- for (auto End : Shape.CoroEnds) {
- replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, CG);
- }
-}
-
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape) {
assert(Shape.ABI == coro::ABI::Async);
@@ -1150,7 +1162,8 @@ static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
Function *DestroyFn, Function *CleanupFn) {
assert(Shape.ABI == coro::ABI::Switch);
- IRBuilder<> Builder(Shape.FramePtr->getNextNode());
+ IRBuilder<> Builder(Shape.getInsertPtAfterFramePtr());
+
auto *ResumeAddr = Builder.CreateStructGEP(
Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Resume,
"resume.addr");
@@ -1559,7 +1572,8 @@ static void simplifySuspendPoints(coro::Shape &Shape) {
}
static void splitSwitchCoroutine(Function &F, coro::Shape &Shape,
- SmallVectorImpl<Function *> &Clones) {
+ SmallVectorImpl<Function *> &Clones,
+ TargetTransformInfo &TTI) {
assert(Shape.ABI == coro::ABI::Switch);
createResumeEntryBlock(F, Shape);
@@ -1574,7 +1588,13 @@ static void splitSwitchCoroutine(Function &F, coro::Shape &Shape,
postSplitCleanup(*DestroyClone);
postSplitCleanup(*CleanupClone);
- addMustTailToCoroResumes(*ResumeClone);
+ // Adding musttail call to support symmetric transfer.
+ // Skip targets which don't support tail call.
+ //
+ // FIXME: Could we support symmetric transfer effectively without musttail
+ // call?
+ if (TTI.supportsTailCalls())
+ addMustTailToCoroResumes(*ResumeClone);
// Store addresses resume/destroy/cleanup functions in the coroutine frame.
updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
@@ -1661,7 +1681,7 @@ static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
// Map all uses of llvm.coro.begin to the allocated frame pointer.
{
// Make sure we don't invalidate Shape.FramePtr.
- TrackingVH<Instruction> Handle(Shape.FramePtr);
+ TrackingVH<Value> Handle(Shape.FramePtr);
Shape.CoroBegin->replaceAllUsesWith(FramePtr);
Shape.FramePtr = Handle.getValPtr();
}
@@ -1773,7 +1793,7 @@ static void splitRetconCoroutine(Function &F, coro::Shape &Shape,
// Map all uses of llvm.coro.begin to the allocated frame pointer.
{
// Make sure we don't invalidate Shape.FramePtr.
- TrackingVH<Instruction> Handle(Shape.FramePtr);
+ TrackingVH<Value> Handle(Shape.FramePtr);
Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
Shape.FramePtr = Handle.getValPtr();
}
@@ -1879,6 +1899,7 @@ namespace {
static coro::Shape splitCoroutine(Function &F,
SmallVectorImpl<Function *> &Clones,
+ TargetTransformInfo &TTI,
bool OptimizeFrame) {
PrettyStackTraceFunction prettyStackTrace(F);
@@ -1901,7 +1922,7 @@ static coro::Shape splitCoroutine(Function &F,
} else {
switch (Shape.ABI) {
case coro::ABI::Switch:
- splitSwitchCoroutine(F, Shape, Clones);
+ splitSwitchCoroutine(F, Shape, Clones, TTI);
break;
case coro::ABI::Async:
splitAsyncCoroutine(F, Shape, Clones);
@@ -1917,21 +1938,27 @@ static coro::Shape splitCoroutine(Function &F,
// This invalidates SwiftErrorOps in the Shape.
replaceSwiftErrorOps(F, Shape, nullptr);
- return Shape;
-}
-
-static void
-updateCallGraphAfterCoroutineSplit(Function &F, const coro::Shape &Shape,
- const SmallVectorImpl<Function *> &Clones,
- CallGraph &CG, CallGraphSCC &SCC) {
- if (!Shape.CoroBegin)
- return;
-
- removeCoroEnds(Shape, &CG);
- postSplitCleanup(F);
+ // Finally, salvage the llvm.dbg.{declare,addr} in our original function that
+ // point into the coroutine frame. We only do this for the current function
+ // since the Cloner salvaged debug info for us in the new coroutine funclets.
+ SmallVector<DbgVariableIntrinsic *, 8> Worklist;
+ SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache;
+ for (auto &BB : F) {
+ for (auto &I : BB) {
+ if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) {
+ Worklist.push_back(DDI);
+ continue;
+ }
+ if (auto *DDI = dyn_cast<DbgAddrIntrinsic>(&I)) {
+ Worklist.push_back(DDI);
+ continue;
+ }
+ }
+ }
+ for (auto *DDI : Worklist)
+ coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
- // Update call graph and add the functions we created to the SCC.
- coro::updateCallGraph(F, Clones, CG, SCC);
+ return Shape;
}
static void updateCallGraphAfterCoroutineSplit(
@@ -1976,70 +2003,6 @@ static void updateCallGraphAfterCoroutineSplit(
updateCGAndAnalysisManagerForFunctionPass(CG, C, N, AM, UR, FAM);
}
-// When we see the coroutine the first time, we insert an indirect call to a
-// devirt trigger function and mark the coroutine that it is now ready for
-// split.
-// Async lowering uses this after it has split the function to restart the
-// pipeline.
-static void prepareForSplit(Function &F, CallGraph &CG,
- bool MarkForAsyncRestart = false) {
- Module &M = *F.getParent();
- LLVMContext &Context = F.getContext();
-#ifndef NDEBUG
- Function *DevirtFn = M.getFunction(CORO_DEVIRT_TRIGGER_FN);
- assert(DevirtFn && "coro.devirt.trigger function not found");
-#endif
-
- F.addFnAttr(CORO_PRESPLIT_ATTR, MarkForAsyncRestart
- ? ASYNC_RESTART_AFTER_SPLIT
- : PREPARED_FOR_SPLIT);
-
- // Insert an indirect call sequence that will be devirtualized by CoroElide
- // pass:
- // %0 = call i8* @llvm.coro.subfn.addr(i8* null, i8 -1)
- // %1 = bitcast i8* %0 to void(i8*)*
- // call void %1(i8* null)
- coro::LowererBase Lowerer(M);
- Instruction *InsertPt =
- MarkForAsyncRestart ? F.getEntryBlock().getFirstNonPHIOrDbgOrLifetime()
- : F.getEntryBlock().getTerminator();
- auto *Null = ConstantPointerNull::get(Type::getInt8PtrTy(Context));
- auto *DevirtFnAddr =
- Lowerer.makeSubFnCall(Null, CoroSubFnInst::RestartTrigger, InsertPt);
- FunctionType *FnTy = FunctionType::get(Type::getVoidTy(Context),
- {Type::getInt8PtrTy(Context)}, false);
- auto *IndirectCall = CallInst::Create(FnTy, DevirtFnAddr, Null, "", InsertPt);
-
- // Update CG graph with an indirect call we just added.
- CG[&F]->addCalledFunction(IndirectCall, CG.getCallsExternalNode());
-}
-
-// Make sure that there is a devirtualization trigger function that the
-// coro-split pass uses to force a restart of the CGSCC pipeline. If the devirt
-// trigger function is not found, we will create one and add it to the current
-// SCC.
-static void createDevirtTriggerFunc(CallGraph &CG, CallGraphSCC &SCC) {
- Module &M = CG.getModule();
- if (M.getFunction(CORO_DEVIRT_TRIGGER_FN))
- return;
-
- LLVMContext &C = M.getContext();
- auto *FnTy = FunctionType::get(Type::getVoidTy(C), Type::getInt8PtrTy(C),
- /*isVarArg=*/false);
- Function *DevirtFn =
- Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
- CORO_DEVIRT_TRIGGER_FN, &M);
- DevirtFn->addFnAttr(Attribute::AlwaysInline);
- auto *Entry = BasicBlock::Create(C, "entry", DevirtFn);
- ReturnInst::Create(C, Entry);
-
- auto *Node = CG.getOrInsertFunction(DevirtFn);
-
- SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end());
- Nodes.push_back(Node);
- SCC.initialize(Nodes);
-}
-
/// Replace a call to llvm.coro.prepare.retcon.
static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
LazyCallGraph::SCC &C) {
@@ -2076,59 +2039,6 @@ static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
Cast->eraseFromParent();
}
}
-/// Replace a call to llvm.coro.prepare.retcon.
-static void replacePrepare(CallInst *Prepare, CallGraph &CG) {
- auto CastFn = Prepare->getArgOperand(0); // as an i8*
- auto Fn = CastFn->stripPointerCasts(); // as its original type
-
- // Find call graph nodes for the preparation.
- CallGraphNode *PrepareUserNode = nullptr, *FnNode = nullptr;
- if (auto ConcreteFn = dyn_cast<Function>(Fn)) {
- PrepareUserNode = CG[Prepare->getFunction()];
- FnNode = CG[ConcreteFn];
- }
-
- // Attempt to peephole this pattern:
- // %0 = bitcast [[TYPE]] @some_function to i8*
- // %1 = call @llvm.coro.prepare.retcon(i8* %0)
- // %2 = bitcast %1 to [[TYPE]]
- // ==>
- // %2 = @some_function
- for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
- // Look for bitcasts back to the original function type.
- auto *Cast = dyn_cast<BitCastInst>(U.getUser());
- if (!Cast || Cast->getType() != Fn->getType()) continue;
-
- // Check whether the replacement will introduce new direct calls.
- // If so, we'll need to update the call graph.
- if (PrepareUserNode) {
- for (auto &Use : Cast->uses()) {
- if (auto *CB = dyn_cast<CallBase>(Use.getUser())) {
- if (!CB->isCallee(&Use))
- continue;
- PrepareUserNode->removeCallEdgeFor(*CB);
- PrepareUserNode->addCalledFunction(CB, FnNode);
- }
- }
- }
-
- // Replace and remove the cast.
- Cast->replaceAllUsesWith(Fn);
- Cast->eraseFromParent();
- }
-
- // Replace any remaining uses with the function as an i8*.
- // This can never directly be a callee, so we don't need to update CG.
- Prepare->replaceAllUsesWith(CastFn);
- Prepare->eraseFromParent();
-
- // Kill dead bitcasts.
- while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
- if (!Cast->use_empty()) break;
- CastFn = Cast->getOperand(0);
- Cast->eraseFromParent();
- }
-}
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
LazyCallGraph::SCC &C) {
@@ -2143,30 +2053,6 @@ static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
return Changed;
}
-/// Remove calls to llvm.coro.prepare.retcon, a barrier meant to prevent
-/// IPO from operating on calls to a retcon coroutine before it's been
-/// split. This is only safe to do after we've split all retcon
-/// coroutines in the module. We can do that this in this pass because
-/// this pass does promise to split all retcon coroutines (as opposed to
-/// switch coroutines, which are lowered in multiple stages).
-static bool replaceAllPrepares(Function *PrepareFn, CallGraph &CG) {
- bool Changed = false;
- for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
- // Intrinsics can only be used in calls.
- auto *Prepare = cast<CallInst>(P.getUser());
- replacePrepare(Prepare, CG);
- Changed = true;
- }
-
- return Changed;
-}
-
-static bool declaresCoroSplitIntrinsics(const Module &M) {
- return coro::declaresIntrinsics(M, {"llvm.coro.begin",
- "llvm.coro.prepare.retcon",
- "llvm.coro.prepare.async"});
-}
-
static void addPrepareFunction(const Module &M,
SmallVectorImpl<Function *> &Fns,
StringRef Name) {
@@ -2185,18 +2071,15 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
auto &FAM =
AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
- if (!declaresCoroSplitIntrinsics(M))
- return PreservedAnalyses::all();
-
// Check for uses of llvm.coro.prepare.retcon/async.
SmallVector<Function *, 2> PrepareFns;
addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
// Find coroutines for processing.
- SmallVector<LazyCallGraph::Node *, 4> Coroutines;
+ SmallVector<LazyCallGraph::Node *> Coroutines;
for (LazyCallGraph::Node &N : C)
- if (N.getFunction().hasFnAttribute(CORO_PRESPLIT_ATTR))
+ if (N.getFunction().isPresplitCoroutine())
Coroutines.push_back(&N);
if (Coroutines.empty() && PrepareFns.empty())
@@ -2212,13 +2095,12 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
for (LazyCallGraph::Node *N : Coroutines) {
Function &F = N->getFunction();
LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
- << "' state: "
- << F.getFnAttribute(CORO_PRESPLIT_ATTR).getValueAsString()
<< "\n");
- F.removeFnAttr(CORO_PRESPLIT_ATTR);
+ F.setSplittedCoroutine();
SmallVector<Function *, 4> Clones;
- const coro::Shape Shape = splitCoroutine(F, Clones, OptimizeFrame);
+ const coro::Shape Shape = splitCoroutine(
+ F, Clones, FAM.getResult<TargetIRAnalysis>(F), OptimizeFrame);
updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
if (!Shape.CoroSuspends.empty()) {
@@ -2237,122 +2119,3 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
return PreservedAnalyses::none();
}
-
-namespace {
-
-// We present a coroutine to LLVM as an ordinary function with suspension
-// points marked up with intrinsics. We let the optimizer party on the coroutine
-// as a single function for as long as possible. Shortly before the coroutine is
-// eligible to be inlined into its callers, we split up the coroutine into parts
-// corresponding to initial, resume and destroy invocations of the coroutine,
-// add them to the current SCC and restart the IPO pipeline to optimize the
-// coroutine subfunctions we extracted before proceeding to the caller of the
-// coroutine.
-struct CoroSplitLegacy : public CallGraphSCCPass {
- static char ID; // Pass identification, replacement for typeid
-
- CoroSplitLegacy(bool OptimizeFrame = false)
- : CallGraphSCCPass(ID), OptimizeFrame(OptimizeFrame) {
- initializeCoroSplitLegacyPass(*PassRegistry::getPassRegistry());
- }
-
- bool Run = false;
- bool OptimizeFrame;
-
- // A coroutine is identified by the presence of coro.begin intrinsic, if
- // we don't have any, this pass has nothing to do.
- bool doInitialization(CallGraph &CG) override {
- Run = declaresCoroSplitIntrinsics(CG.getModule());
- return CallGraphSCCPass::doInitialization(CG);
- }
-
- bool runOnSCC(CallGraphSCC &SCC) override {
- if (!Run)
- return false;
-
- // Check for uses of llvm.coro.prepare.retcon.
- SmallVector<Function *, 2> PrepareFns;
- auto &M = SCC.getCallGraph().getModule();
- addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
- addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
-
- // Find coroutines for processing.
- SmallVector<Function *, 4> Coroutines;
- for (CallGraphNode *CGN : SCC)
- if (auto *F = CGN->getFunction())
- if (F->hasFnAttribute(CORO_PRESPLIT_ATTR))
- Coroutines.push_back(F);
-
- if (Coroutines.empty() && PrepareFns.empty())
- return false;
-
- CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
-
- if (Coroutines.empty()) {
- bool Changed = false;
- for (auto *PrepareFn : PrepareFns)
- Changed |= replaceAllPrepares(PrepareFn, CG);
- return Changed;
- }
-
- createDevirtTriggerFunc(CG, SCC);
-
- // Split all the coroutines.
- for (Function *F : Coroutines) {
- Attribute Attr = F->getFnAttribute(CORO_PRESPLIT_ATTR);
- StringRef Value = Attr.getValueAsString();
- LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F->getName()
- << "' state: " << Value << "\n");
- // Async lowering marks coroutines to trigger a restart of the pipeline
- // after it has split them.
- if (Value == ASYNC_RESTART_AFTER_SPLIT) {
- F->removeFnAttr(CORO_PRESPLIT_ATTR);
- continue;
- }
- if (Value == UNPREPARED_FOR_SPLIT) {
- prepareForSplit(*F, CG);
- continue;
- }
- F->removeFnAttr(CORO_PRESPLIT_ATTR);
-
- SmallVector<Function *, 4> Clones;
- const coro::Shape Shape = splitCoroutine(*F, Clones, OptimizeFrame);
- updateCallGraphAfterCoroutineSplit(*F, Shape, Clones, CG, SCC);
- if (Shape.ABI == coro::ABI::Async) {
- // Restart SCC passes.
- // Mark function for CoroElide pass. It will devirtualize causing a
- // restart of the SCC pipeline.
- prepareForSplit(*F, CG, true /*MarkForAsyncRestart*/);
- }
- }
-
- for (auto *PrepareFn : PrepareFns)
- replaceAllPrepares(PrepareFn, CG);
-
- return true;
- }
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- CallGraphSCCPass::getAnalysisUsage(AU);
- }
-
- StringRef getPassName() const override { return "Coroutine Splitting"; }
-};
-
-} // end anonymous namespace
-
-char CoroSplitLegacy::ID = 0;
-
-INITIALIZE_PASS_BEGIN(
- CoroSplitLegacy, "coro-split",
- "Split coroutine into a set of functions driving its state machine", false,
- false)
-INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
-INITIALIZE_PASS_END(
- CoroSplitLegacy, "coro-split",
- "Split coroutine into a set of functions driving its state machine", false,
- false)
-
-Pass *llvm::createCoroSplitLegacyPass(bool OptimizeFrame) {
- return new CoroSplitLegacy(OptimizeFrame);
-}
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 965a146c143f..1742e9319c3b 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -10,14 +10,11 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/Coroutines.h"
#include "CoroInstr.h"
#include "CoroInternal.h"
-#include "llvm-c/Transforms/Coroutines.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/CallGraph.h"
-#include "llvm/Analysis/CallGraphSCCPass.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
@@ -26,14 +23,10 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/LegacyPassManager.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
-#include "llvm/InitializePasses.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Transforms/IPO.h"
-#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstddef>
@@ -41,55 +34,6 @@
using namespace llvm;
-void llvm::initializeCoroutines(PassRegistry &Registry) {
- initializeCoroEarlyLegacyPass(Registry);
- initializeCoroSplitLegacyPass(Registry);
- initializeCoroElideLegacyPass(Registry);
- initializeCoroCleanupLegacyPass(Registry);
-}
-
-static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createCoroSplitLegacyPass());
- PM.add(createCoroElideLegacyPass());
-
- PM.add(createBarrierNoopPass());
- PM.add(createCoroCleanupLegacyPass());
-}
-
-static void addCoroutineEarlyPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createCoroEarlyLegacyPass());
-}
-
-static void addCoroutineScalarOptimizerPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createCoroElideLegacyPass());
-}
-
-static void addCoroutineSCCPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createCoroSplitLegacyPass(Builder.OptLevel != 0));
-}
-
-static void addCoroutineOptimizerLastPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createCoroCleanupLegacyPass());
-}
-
-void llvm::addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder) {
- Builder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
- addCoroutineEarlyPasses);
- Builder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addCoroutineOpt0Passes);
- Builder.addExtension(PassManagerBuilder::EP_CGSCCOptimizerLate,
- addCoroutineSCCPasses);
- Builder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate,
- addCoroutineScalarOptimizerPasses);
- Builder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addCoroutineOptimizerLastPasses);
-}
-
// Construct the lowerer base class and initialize its members.
coro::LowererBase::LowererBase(Module &M)
: TheModule(M), Context(M.getContext()),
@@ -119,44 +63,55 @@ Value *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
return Bitcast;
}
+// NOTE: Must be sorted!
+static const char *const CoroIntrinsics[] = {
+ "llvm.coro.align",
+ "llvm.coro.alloc",
+ "llvm.coro.async.context.alloc",
+ "llvm.coro.async.context.dealloc",
+ "llvm.coro.async.resume",
+ "llvm.coro.async.size.replace",
+ "llvm.coro.async.store_resume",
+ "llvm.coro.begin",
+ "llvm.coro.destroy",
+ "llvm.coro.done",
+ "llvm.coro.end",
+ "llvm.coro.end.async",
+ "llvm.coro.frame",
+ "llvm.coro.free",
+ "llvm.coro.id",
+ "llvm.coro.id.async",
+ "llvm.coro.id.retcon",
+ "llvm.coro.id.retcon.once",
+ "llvm.coro.noop",
+ "llvm.coro.prepare.async",
+ "llvm.coro.prepare.retcon",
+ "llvm.coro.promise",
+ "llvm.coro.resume",
+ "llvm.coro.save",
+ "llvm.coro.size",
+ "llvm.coro.subfn.addr",
+ "llvm.coro.suspend",
+ "llvm.coro.suspend.async",
+ "llvm.coro.suspend.retcon",
+};
+
#ifndef NDEBUG
static bool isCoroutineIntrinsicName(StringRef Name) {
- // NOTE: Must be sorted!
- static const char *const CoroIntrinsics[] = {
- "llvm.coro.align",
- "llvm.coro.alloc",
- "llvm.coro.async.context.alloc",
- "llvm.coro.async.context.dealloc",
- "llvm.coro.async.resume",
- "llvm.coro.async.size.replace",
- "llvm.coro.async.store_resume",
- "llvm.coro.begin",
- "llvm.coro.destroy",
- "llvm.coro.done",
- "llvm.coro.end",
- "llvm.coro.end.async",
- "llvm.coro.frame",
- "llvm.coro.free",
- "llvm.coro.id",
- "llvm.coro.id.async",
- "llvm.coro.id.retcon",
- "llvm.coro.id.retcon.once",
- "llvm.coro.noop",
- "llvm.coro.prepare.async",
- "llvm.coro.prepare.retcon",
- "llvm.coro.promise",
- "llvm.coro.resume",
- "llvm.coro.save",
- "llvm.coro.size",
- "llvm.coro.subfn.addr",
- "llvm.coro.suspend",
- "llvm.coro.suspend.async",
- "llvm.coro.suspend.retcon",
- };
return Intrinsic::lookupLLVMIntrinsicByName(CoroIntrinsics, Name) != -1;
}
#endif
+bool coro::declaresAnyIntrinsic(const Module &M) {
+ for (StringRef Name : CoroIntrinsics) {
+ assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
+ if (M.getNamedValue(Name))
+ return true;
+ }
+
+ return false;
+}
+
// Verifies if a module has named values listed. Also, in debug mode verifies
// that names are intrinsic names.
bool coro::declaresIntrinsics(const Module &M,
@@ -191,46 +146,6 @@ void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) {
}
}
-// FIXME: This code is stolen from CallGraph::addToCallGraph(Function *F), which
-// happens to be private. It is better for this functionality exposed by the
-// CallGraph.
-static void buildCGN(CallGraph &CG, CallGraphNode *Node) {
- Function *F = Node->getFunction();
-
- // Look for calls by this function.
- for (Instruction &I : instructions(F))
- if (auto *Call = dyn_cast<CallBase>(&I)) {
- const Function *Callee = Call->getCalledFunction();
- if (!Callee || !Intrinsic::isLeaf(Callee->getIntrinsicID()))
- // Indirect calls of intrinsics are not allowed so no need to check.
- // We can be more precise here by using TargetArg returned by
- // Intrinsic::isLeaf.
- Node->addCalledFunction(Call, CG.getCallsExternalNode());
- else if (!Callee->isIntrinsic())
- Node->addCalledFunction(Call, CG.getOrInsertFunction(Callee));
- }
-}
-
-// Rebuild CGN after we extracted parts of the code from ParentFunc into
-// NewFuncs. Builds CGNs for the NewFuncs and adds them to the current SCC.
-void coro::updateCallGraph(Function &ParentFunc, ArrayRef<Function *> NewFuncs,
- CallGraph &CG, CallGraphSCC &SCC) {
- // Rebuild CGN from scratch for the ParentFunc
- auto *ParentNode = CG[&ParentFunc];
- ParentNode->removeAllCalledFunctions();
- buildCGN(CG, ParentNode);
-
- SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end());
-
- for (Function *F : NewFuncs) {
- CallGraphNode *Callee = CG.getOrInsertFunction(F);
- Nodes.push_back(Callee);
- buildCGN(CG, Callee);
- }
-
- SCC.initialize(Nodes);
-}
-
static void clear(coro::Shape &Shape) {
Shape.CoroBegin = nullptr;
Shape.CoroEnds.clear();
@@ -735,25 +650,3 @@ void CoroAsyncEndInst::checkWellFormed() const {
"match the tail arguments",
MustTailCallFunc);
}
-
-void LLVMAddCoroEarlyPass(LLVMPassManagerRef PM) {
- unwrap(PM)->add(createCoroEarlyLegacyPass());
-}
-
-void LLVMAddCoroSplitPass(LLVMPassManagerRef PM) {
- unwrap(PM)->add(createCoroSplitLegacyPass());
-}
-
-void LLVMAddCoroElidePass(LLVMPassManagerRef PM) {
- unwrap(PM)->add(createCoroElideLegacyPass());
-}
-
-void LLVMAddCoroCleanupPass(LLVMPassManagerRef PM) {
- unwrap(PM)->add(createCoroCleanupLegacyPass());
-}
-
-void
-LLVMPassManagerBuilderAddCoroutinePassesToExtensionPoints(LLVMPassManagerBuilderRef PMB) {
- PassManagerBuilder *Builder = unwrap(PMB);
- addCoroutinePassesToExtensionPoints(*Builder);
-}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-31 21:14:51 +0000