vendor/llvm-project/llvmorg-18-init-16595-g7c00a5be5cde

112 files changed, 1220 insertions, 537 deletions

diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index 4a952ccae7a0..8afe455e2f08 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -411,6 +411,7 @@ class AliasSetsPrinterPass : public PassInfoMixin<AliasSetsPrinterPass> {
 public:
   explicit AliasSetsPrinterPass(raw_ostream &OS);
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/AssumptionCache.h b/llvm/include/llvm/Analysis/AssumptionCache.h
index 12dd9b04c932..96ae32da6743 100644
--- a/llvm/include/llvm/Analysis/AssumptionCache.h
+++ b/llvm/include/llvm/Analysis/AssumptionCache.h
@@ -189,6 +189,8 @@ public:
   explicit AssumptionPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// An immutable pass that tracks lazily created \c AssumptionCache
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
index 95d75b0e1854..179fd06addec 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -134,6 +134,8 @@ public:
   explicit BlockFrequencyPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Legacy analysis pass which computes \c BlockFrequencyInfo.
diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index fb02997371bf..6b9d17818201 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -436,6 +436,8 @@ public:
   explicit BranchProbabilityPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Legacy analysis pass which computes \c BranchProbabilityInfo.
diff --git a/llvm/include/llvm/Analysis/CFGSCCPrinter.h b/llvm/include/llvm/Analysis/CFGSCCPrinter.h
index d98071461f57..0ea0b46c4626 100644
--- a/llvm/include/llvm/Analysis/CFGSCCPrinter.h
+++ b/llvm/include/llvm/Analysis/CFGSCCPrinter.h
@@ -19,6 +19,7 @@ class CFGSCCPrinterPass : public PassInfoMixin<CFGSCCPrinterPass> {
 public:
   explicit CFGSCCPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/CallGraph.h b/llvm/include/llvm/Analysis/CallGraph.h
index 9413b39978e3..887743774175 100644
--- a/llvm/include/llvm/Analysis/CallGraph.h
+++ b/llvm/include/llvm/Analysis/CallGraph.h
@@ -322,6 +322,8 @@ public:
   explicit CallGraphPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Printer pass for the summarized \c CallGraphAnalysis results.
@@ -333,6 +335,8 @@ public:
   explicit CallGraphSCCsPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// The \c ModulePass which wraps up a \c CallGraph and the logic to
diff --git a/llvm/include/llvm/Analysis/CallPrinter.h b/llvm/include/llvm/Analysis/CallPrinter.h
index d325d0010371..95cb5cc3ca86 100644
--- a/llvm/include/llvm/Analysis/CallPrinter.h
+++ b/llvm/include/llvm/Analysis/CallPrinter.h
@@ -24,12 +24,14 @@ class ModulePass;
 class CallGraphDOTPrinterPass : public PassInfoMixin<CallGraphDOTPrinterPass> {
 public:
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Pass for viewing the call graph
 class CallGraphViewerPass : public PassInfoMixin<CallGraphViewerPass> {
 public:
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 ModulePass *createCallGraphViewerPass();
diff --git a/llvm/include/llvm/Analysis/CostModel.h b/llvm/include/llvm/Analysis/CostModel.h
index 649168050cec..9b127c27ba7e 100644
--- a/llvm/include/llvm/Analysis/CostModel.h
+++ b/llvm/include/llvm/Analysis/CostModel.h
@@ -20,6 +20,8 @@ public:
   explicit CostModelPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/CycleAnalysis.h b/llvm/include/llvm/Analysis/CycleAnalysis.h
index 099d7611dedc..ce939eff8ff8 100644
--- a/llvm/include/llvm/Analysis/CycleAnalysis.h
+++ b/llvm/include/llvm/Analysis/CycleAnalysis.h
@@ -68,6 +68,8 @@ public:
   explicit CycleInfoPrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/DDG.h b/llvm/include/llvm/Analysis/DDG.h
index bc599cb1f9a1..bd559f3fb69b 100644
--- a/llvm/include/llvm/Analysis/DDG.h
+++ b/llvm/include/llvm/Analysis/DDG.h
@@ -427,6 +427,7 @@ public:
   explicit DDGAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &U);
+  static bool isRequired() { return true; }
 
 private:
   raw_ostream &OS;
diff --git a/llvm/include/llvm/Analysis/DDGPrinter.h b/llvm/include/llvm/Analysis/DDGPrinter.h
index d93c28280bac..4aa154d173ba 100644
--- a/llvm/include/llvm/Analysis/DDGPrinter.h
+++ b/llvm/include/llvm/Analysis/DDGPrinter.h
@@ -29,6 +29,7 @@ class DDGDotPrinterPass : public PassInfoMixin<DDGDotPrinterPass> {
 public:
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &U);
+  static bool isRequired() { return true; }
 };
 
 //===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Analysis/Delinearization.h b/llvm/include/llvm/Analysis/Delinearization.h
index 95a36b8b79a4..a00adb289604 100644
--- a/llvm/include/llvm/Analysis/Delinearization.h
+++ b/llvm/include/llvm/Analysis/Delinearization.h
@@ -140,6 +140,7 @@ struct DelinearizationPrinterPass
     : public PassInfoMixin<DelinearizationPrinterPass> {
   explicit DelinearizationPrinterPass(raw_ostream &OS);
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 
 private:
   raw_ostream &OS;
diff --git a/llvm/include/llvm/Analysis/DemandedBits.h b/llvm/include/llvm/Analysis/DemandedBits.h
index 6e4bfcf899c9..aac7382528f0 100644
--- a/llvm/include/llvm/Analysis/DemandedBits.h
+++ b/llvm/include/llvm/Analysis/DemandedBits.h
@@ -120,6 +120,8 @@ public:
   explicit DemandedBitsPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h
index 327315f831e1..f0a09644e0f4 100644
--- a/llvm/include/llvm/Analysis/DependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h
@@ -994,6 +994,8 @@ namespace llvm {
 
     PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
 
+    static bool isRequired() { return true; }
+
   private:
     raw_ostream &OS;
     bool NormalizeResults;
diff --git a/llvm/include/llvm/Analysis/DominanceFrontier.h b/llvm/include/llvm/Analysis/DominanceFrontier.h
index db0130e4804b..b65cdc9cdb3c 100644
--- a/llvm/include/llvm/Analysis/DominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/DominanceFrontier.h
@@ -204,6 +204,8 @@ public:
   explicit DominanceFrontierPrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
index 3e9eb9374563..f5fbbdcb7143 100644
--- a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
+++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
@@ -157,6 +157,8 @@ public:
   explicit FunctionPropertiesPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Correctly update FunctionPropertiesInfo post-inlining. A
diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
index ad137baff5d4..0d19de6edc2a 100644
--- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
+++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
@@ -1198,6 +1198,7 @@ class IRSimilarityAnalysisPrinterPass
 public:
   explicit IRSimilarityAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 2740106bc7db..5f36ee6f68ab 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -341,7 +341,7 @@ public:
   Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
 };
 
-/// Printer pass for the FunctionPropertiesAnalysis results.
+/// Printer pass for the InlineAdvisorAnalysis results.
 class InlineAdvisorAnalysisPrinterPass
     : public PassInfoMixin<InlineAdvisorAnalysisPrinterPass> {
   raw_ostream &OS;
@@ -353,6 +353,7 @@ public:
 
   PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM,
                         LazyCallGraph &CG, CGSCCUpdateResult &UR);
+  static bool isRequired() { return true; }
 };
 
 std::unique_ptr<InlineAdvisor>
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index 3f0bb879e021..3a760e0a85ce 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -343,6 +343,7 @@ struct InlineCostAnnotationPrinterPass
 public:
   explicit InlineCostAnnotationPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+  static bool isRequired() { return true; }
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/InlineSizeEstimatorAnalysis.h b/llvm/include/llvm/Analysis/InlineSizeEstimatorAnalysis.h
index 0aae696a98a9..b44edd370dd1 100644
--- a/llvm/include/llvm/Analysis/InlineSizeEstimatorAnalysis.h
+++ b/llvm/include/llvm/Analysis/InlineSizeEstimatorAnalysis.h
@@ -40,6 +40,8 @@ public:
   explicit InlineSizeEstimatorAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 } // namespace llvm
 #endif // LLVM_ANALYSIS_INLINESIZEESTIMATORANALYSIS_H
diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h
index 211a058aa017..68c98b416ce9 100644
--- a/llvm/include/llvm/Analysis/LazyCallGraph.h
+++ b/llvm/include/llvm/Analysis/LazyCallGraph.h
@@ -1288,6 +1288,8 @@ public:
   explicit LazyCallGraphPrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// A pass which prints the call graph as a DOT file to a \c raw_ostream.
@@ -1301,6 +1303,8 @@ public:
   explicit LazyCallGraphDOTPrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h
index 25a2c9ffa534..5611a2b98020 100644
--- a/llvm/include/llvm/Analysis/LazyValueInfo.h
+++ b/llvm/include/llvm/Analysis/LazyValueInfo.h
@@ -157,6 +157,8 @@ public:
   explicit LazyValueInfoPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Wrapper around LazyValueInfo.
diff --git a/llvm/include/llvm/Analysis/LoopCacheAnalysis.h b/llvm/include/llvm/Analysis/LoopCacheAnalysis.h
index c9e853b9be8e..4fd2485e39d6 100644
--- a/llvm/include/llvm/Analysis/LoopCacheAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopCacheAnalysis.h
@@ -291,6 +291,8 @@ public:
 
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &U);
+
+  static bool isRequired() { return true; }
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index 3b106381fbca..52084630560c 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -580,11 +580,13 @@ class LoopPrinterPass : public PassInfoMixin<LoopPrinterPass> {
 public:
   explicit LoopPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Verifier pass for the \c LoopAnalysis results.
 struct LoopVerifierPass : public PassInfoMixin<LoopVerifierPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// The legacy pass manager's analysis pass to compute loop information.
diff --git a/llvm/include/llvm/Analysis/LoopNestAnalysis.h b/llvm/include/llvm/Analysis/LoopNestAnalysis.h
index 852a6c438d43..3b33dd505dde 100644
--- a/llvm/include/llvm/Analysis/LoopNestAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopNestAnalysis.h
@@ -217,6 +217,8 @@ public:
 
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &U);
+
+  static bool isRequired() { return true; }
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Analysis/MemDerefPrinter.h b/llvm/include/llvm/Analysis/MemDerefPrinter.h
index bafdc543eeaf..ba376dadb2a7 100644
--- a/llvm/include/llvm/Analysis/MemDerefPrinter.h
+++ b/llvm/include/llvm/Analysis/MemDerefPrinter.h
@@ -18,6 +18,7 @@ class MemDerefPrinterPass : public PassInfoMixin<MemDerefPrinterPass> {
 public:
   MemDerefPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index 827b5081b2ce..37ce1518f00c 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -187,80 +187,119 @@ Value *lowerObjectSizeCall(
     const TargetLibraryInfo *TLI, AAResults *AA, bool MustSucceed,
     SmallVectorImpl<Instruction *> *InsertedInstructions = nullptr);
 
-using SizeOffsetType = std::pair<APInt, APInt>;
+/// SizeOffsetType - A base template class for the object size visitors. Used
+/// here as a self-documenting way to handle the values rather than using a
+/// \p std::pair.
+template <typename T, class C> struct SizeOffsetType {
+public:
+  T Size;
+  T Offset;
+
+  SizeOffsetType() = default;
+  SizeOffsetType(T Size, T Offset) : Size(Size), Offset(Offset) {}
+
+  bool knownSize() const { return C::known(Size); }
+  bool knownOffset() const { return C::known(Offset); }
+  bool anyKnown() const { return knownSize() || knownOffset(); }
+  bool bothKnown() const { return knownSize() && knownOffset(); }
+
+  bool operator==(const SizeOffsetType<T, C> &RHS) const {
+    return Size == RHS.Size && Offset == RHS.Offset;
+  }
+  bool operator!=(const SizeOffsetType<T, C> &RHS) const {
+    return !(*this == RHS);
+  }
+};
+
+/// SizeOffsetAPInt - Used by \p ObjectSizeOffsetVisitor, which works with
+/// \p APInts.
+struct SizeOffsetAPInt : public SizeOffsetType<APInt, SizeOffsetAPInt> {
+  SizeOffsetAPInt() = default;
+  SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {}
+
+  static bool known(APInt V) { return V.getBitWidth() > 1; }
+};
 
 /// Evaluate the size and offset of an object pointed to by a Value*
 /// statically. Fails if size or offset are not known at compile time.
 class ObjectSizeOffsetVisitor
-  : public InstVisitor<ObjectSizeOffsetVisitor, SizeOffsetType> {
+    : public InstVisitor<ObjectSizeOffsetVisitor, SizeOffsetAPInt> {
   const DataLayout &DL;
   const TargetLibraryInfo *TLI;
   ObjectSizeOpts Options;
   unsigned IntTyBits;
   APInt Zero;
-  SmallDenseMap<Instruction *, SizeOffsetType, 8> SeenInsts;
+  SmallDenseMap<Instruction *, SizeOffsetAPInt, 8> SeenInsts;
   unsigned InstructionsVisited;
 
   APInt align(APInt Size, MaybeAlign Align);
 
-  SizeOffsetType unknown() {
-    return std::make_pair(APInt(), APInt());
-  }
+  static SizeOffsetAPInt unknown() { return SizeOffsetAPInt(); }
 
 public:
   ObjectSizeOffsetVisitor(const DataLayout &DL, const TargetLibraryInfo *TLI,
                           LLVMContext &Context, ObjectSizeOpts Options = {});
 
-  SizeOffsetType compute(Value *V);
-
-  static bool knownSize(const SizeOffsetType &SizeOffset) {
-    return SizeOffset.first.getBitWidth() > 1;
-  }
-
-  static bool knownOffset(const SizeOffsetType &SizeOffset) {
-    return SizeOffset.second.getBitWidth() > 1;
-  }
-
-  static bool bothKnown(const SizeOffsetType &SizeOffset) {
-    return knownSize(SizeOffset) && knownOffset(SizeOffset);
-  }
+  SizeOffsetAPInt compute(Value *V);
 
   // These are "private", except they can't actually be made private. Only
   // compute() should be used by external users.
-  SizeOffsetType visitAllocaInst(AllocaInst &I);
-  SizeOffsetType visitArgument(Argument &A);
-  SizeOffsetType visitCallBase(CallBase &CB);
-  SizeOffsetType visitConstantPointerNull(ConstantPointerNull&);
-  SizeOffsetType visitExtractElementInst(ExtractElementInst &I);
-  SizeOffsetType visitExtractValueInst(ExtractValueInst &I);
-  SizeOffsetType visitGlobalAlias(GlobalAlias &GA);
-  SizeOffsetType visitGlobalVariable(GlobalVariable &GV);
-  SizeOffsetType visitIntToPtrInst(IntToPtrInst&);
-  SizeOffsetType visitLoadInst(LoadInst &I);
-  SizeOffsetType visitPHINode(PHINode&);
-  SizeOffsetType visitSelectInst(SelectInst &I);
-  SizeOffsetType visitUndefValue(UndefValue&);
-  SizeOffsetType visitInstruction(Instruction &I);
+  SizeOffsetAPInt visitAllocaInst(AllocaInst &I);
+  SizeOffsetAPInt visitArgument(Argument &A);
+  SizeOffsetAPInt visitCallBase(CallBase &CB);
+  SizeOffsetAPInt visitConstantPointerNull(ConstantPointerNull &);
+  SizeOffsetAPInt visitExtractElementInst(ExtractElementInst &I);
+  SizeOffsetAPInt visitExtractValueInst(ExtractValueInst &I);
+  SizeOffsetAPInt visitGlobalAlias(GlobalAlias &GA);
+  SizeOffsetAPInt visitGlobalVariable(GlobalVariable &GV);
+  SizeOffsetAPInt visitIntToPtrInst(IntToPtrInst &);
+  SizeOffsetAPInt visitLoadInst(LoadInst &I);
+  SizeOffsetAPInt visitPHINode(PHINode &);
+  SizeOffsetAPInt visitSelectInst(SelectInst &I);
+  SizeOffsetAPInt visitUndefValue(UndefValue &);
+  SizeOffsetAPInt visitInstruction(Instruction &I);
 
 private:
-  SizeOffsetType findLoadSizeOffset(
+  SizeOffsetAPInt findLoadSizeOffset(
       LoadInst &LoadFrom, BasicBlock &BB, BasicBlock::iterator From,
-      SmallDenseMap<BasicBlock *, SizeOffsetType, 8> &VisitedBlocks,
+      SmallDenseMap<BasicBlock *, SizeOffsetAPInt, 8> &VisitedBlocks,
       unsigned &ScannedInstCount);
-  SizeOffsetType combineSizeOffset(SizeOffsetType LHS, SizeOffsetType RHS);
-  SizeOffsetType computeImpl(Value *V);
-  SizeOffsetType computeValue(Value *V);
+  SizeOffsetAPInt combineSizeOffset(SizeOffsetAPInt LHS, SizeOffsetAPInt RHS);
+  SizeOffsetAPInt computeImpl(Value *V);
+  SizeOffsetAPInt computeValue(Value *V);
   bool CheckedZextOrTrunc(APInt &I);
 };
 
-using SizeOffsetEvalType = std::pair<Value *, Value *>;
+/// SizeOffsetValue - Used by \p ObjectSizeOffsetEvaluator, which works with
+/// \p Values.
+struct SizeOffsetWeakTrackingVH;
+struct SizeOffsetValue : public SizeOffsetType<Value *, SizeOffsetValue> {
+  SizeOffsetValue() : SizeOffsetType(nullptr, nullptr) {}
+  SizeOffsetValue(Value *Size, Value *Offset) : SizeOffsetType(Size, Offset) {}
+  SizeOffsetValue(const SizeOffsetWeakTrackingVH &SOT);
+
+  static bool known(Value *V) { return V != nullptr; }
+};
+
+/// SizeOffsetWeakTrackingVH - Used by \p ObjectSizeOffsetEvaluator in a
+/// \p DenseMap.
+struct SizeOffsetWeakTrackingVH
+    : public SizeOffsetType<WeakTrackingVH, SizeOffsetWeakTrackingVH> {
+  SizeOffsetWeakTrackingVH() : SizeOffsetType(nullptr, nullptr) {}
+  SizeOffsetWeakTrackingVH(Value *Size, Value *Offset)
+      : SizeOffsetType(Size, Offset) {}
+  SizeOffsetWeakTrackingVH(const SizeOffsetValue &SOV)
+      : SizeOffsetType(SOV.Size, SOV.Offset) {}
+
+  static bool known(WeakTrackingVH V) { return V.pointsToAliveValue(); }
+};
 
 /// Evaluate the size and offset of an object pointed to by a Value*.
 /// May create code to compute the result at run-time.
 class ObjectSizeOffsetEvaluator
-  : public InstVisitor<ObjectSizeOffsetEvaluator, SizeOffsetEvalType> {
+    : public InstVisitor<ObjectSizeOffsetEvaluator, SizeOffsetValue> {
   using BuilderTy = IRBuilder<TargetFolder, IRBuilderCallbackInserter>;
-  using WeakEvalType = std::pair<WeakTrackingVH, WeakTrackingVH>;
+  using WeakEvalType = SizeOffsetWeakTrackingVH;
   using CacheMapTy = DenseMap<const Value *, WeakEvalType>;
   using PtrSetTy = SmallPtrSet<const Value *, 8>;
 
@@ -275,45 +314,27 @@ class ObjectSizeOffsetEvaluator
   ObjectSizeOpts EvalOpts;
   SmallPtrSet<Instruction *, 8> InsertedInstructions;
 
-  SizeOffsetEvalType compute_(Value *V);
+  SizeOffsetValue compute_(Value *V);
 
 public:
-  static SizeOffsetEvalType unknown() {
-    return std::make_pair(nullptr, nullptr);
-  }
-
   ObjectSizeOffsetEvaluator(const DataLayout &DL, const TargetLibraryInfo *TLI,
                             LLVMContext &Context, ObjectSizeOpts EvalOpts = {});
 
-  SizeOffsetEvalType compute(Value *V);
+  static SizeOffsetValue unknown() { return SizeOffsetValue(); }
 
-  bool knownSize(SizeOffsetEvalType SizeOffset) {
-    return SizeOffset.first;
-  }
-
-  bool knownOffset(SizeOffsetEvalType SizeOffset) {
-    return SizeOffset.second;
-  }
-
-  bool anyKnown(SizeOffsetEvalType SizeOffset) {
-    return knownSize(SizeOffset) || knownOffset(SizeOffset);
-  }
-
-  bool bothKnown(SizeOffsetEvalType SizeOffset) {
-    return knownSize(SizeOffset) && knownOffset(SizeOffset);
-  }
+  SizeOffsetValue compute(Value *V);
 
   // The individual instruction visitors should be treated as private.
-  SizeOffsetEvalType visitAllocaInst(AllocaInst &I);
-  SizeOffsetEvalType visitCallBase(CallBase &CB);
-  SizeOffsetEvalType visitExtractElementInst(ExtractElementInst &I);
-  SizeOffsetEvalType visitExtractValueInst(ExtractValueInst &I);
-  SizeOffsetEvalType visitGEPOperator(GEPOperator &GEP);
-  SizeOffsetEvalType visitIntToPtrInst(IntToPtrInst&);
-  SizeOffsetEvalType visitLoadInst(LoadInst &I);
-  SizeOffsetEvalType visitPHINode(PHINode &PHI);
-  SizeOffsetEvalType visitSelectInst(SelectInst &I);
-  SizeOffsetEvalType visitInstruction(Instruction &I);
+  SizeOffsetValue visitAllocaInst(AllocaInst &I);
+  SizeOffsetValue visitCallBase(CallBase &CB);
+  SizeOffsetValue visitExtractElementInst(ExtractElementInst &I);
+  SizeOffsetValue visitExtractValueInst(ExtractValueInst &I);
+  SizeOffsetValue visitGEPOperator(GEPOperator &GEP);
+  SizeOffsetValue visitIntToPtrInst(IntToPtrInst &);
+  SizeOffsetValue visitLoadInst(LoadInst &I);
+  SizeOffsetValue visitPHINode(PHINode &PHI);
+  SizeOffsetValue visitSelectInst(SelectInst &I);
+  SizeOffsetValue visitInstruction(Instruction &I);
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h
index 94d7f1a78b84..caf0e31fd37d 100644
--- a/llvm/include/llvm/Analysis/MemorySSA.h
+++ b/llvm/include/llvm/Analysis/MemorySSA.h
@@ -953,6 +953,8 @@ public:
       : OS(OS), EnsureOptimizedUses(EnsureOptimizedUses) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Printer pass for \c MemorySSA via the walker.
@@ -964,11 +966,14 @@ public:
   explicit MemorySSAWalkerPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Verifier pass for \c MemorySSA.
 struct MemorySSAVerifierPass : PassInfoMixin<MemorySSAVerifierPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Legacy analysis pass which computes \c MemorySSA.
diff --git a/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h b/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h
index fa91e4f653d0..e69db780a206 100644
--- a/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h
+++ b/llvm/include/llvm/Analysis/ModuleDebugInfoPrinter.h
@@ -23,6 +23,7 @@ class ModuleDebugInfoPrinterPass
 public:
   explicit ModuleDebugInfoPrinterPass(raw_ostream &OS);
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/MustExecute.h b/llvm/include/llvm/Analysis/MustExecute.h
index 9c97bd1725ac..468d94e7cd68 100644
--- a/llvm/include/llvm/Analysis/MustExecute.h
+++ b/llvm/include/llvm/Analysis/MustExecute.h
@@ -547,6 +547,7 @@ class MustExecutePrinterPass : public PassInfoMixin<MustExecutePrinterPass> {
 public:
   MustExecutePrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 class MustBeExecutedContextPrinterPass
@@ -556,6 +557,7 @@ class MustBeExecutedContextPrinterPass
 public:
   MustBeExecutedContextPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Analysis/PhiValues.h b/llvm/include/llvm/Analysis/PhiValues.h
index ecbb8874b378..a749af30be9e 100644
--- a/llvm/include/llvm/Analysis/PhiValues.h
+++ b/llvm/include/llvm/Analysis/PhiValues.h
@@ -132,6 +132,7 @@ class PhiValuesPrinterPass : public PassInfoMixin<PhiValuesPrinterPass> {
 public:
   explicit PhiValuesPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Wrapper pass for the legacy pass manager
diff --git a/llvm/include/llvm/Analysis/PostDominators.h b/llvm/include/llvm/Analysis/PostDominators.h
index 4383113c8db1..92e30f82501c 100644
--- a/llvm/include/llvm/Analysis/PostDominators.h
+++ b/llvm/include/llvm/Analysis/PostDominators.h
@@ -68,6 +68,8 @@ public:
   explicit PostDominatorTreePrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 struct PostDominatorTreeWrapperPass : public FunctionPass {
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index e49538bfaf80..73be9e1d74a3 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -389,6 +389,7 @@ class ProfileSummaryPrinterPass
 public:
   explicit ProfileSummaryPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Analysis/RegionInfo.h b/llvm/include/llvm/Analysis/RegionInfo.h
index 612b977f1ffa..fc8df36ec287 100644
--- a/llvm/include/llvm/Analysis/RegionInfo.h
+++ b/llvm/include/llvm/Analysis/RegionInfo.h
@@ -983,11 +983,14 @@ public:
   explicit RegionInfoPrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Verifier pass for the \c RegionInfo.
 struct RegionInfoVerifierPass : PassInfoMixin<RegionInfoVerifierPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 template <>
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 4f1237c4b1f9..af3ad822e0b0 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -2246,6 +2246,7 @@ class ScalarEvolutionVerifierPass
     : public PassInfoMixin<ScalarEvolutionVerifierPass> {
 public:
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Printer pass for the \c ScalarEvolutionAnalysis results.
@@ -2257,6 +2258,8 @@ public:
   explicit ScalarEvolutionPrinterPass(raw_ostream &OS) : OS(OS) {}
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 class ScalarEvolutionWrapperPass : public FunctionPass {
diff --git a/llvm/include/llvm/Analysis/StackLifetime.h b/llvm/include/llvm/Analysis/StackLifetime.h
index 7fd88362276a..438407fb7056 100644
--- a/llvm/include/llvm/Analysis/StackLifetime.h
+++ b/llvm/include/llvm/Analysis/StackLifetime.h
@@ -190,6 +190,7 @@ public:
   StackLifetimePrinterPass(raw_ostream &OS, StackLifetime::LivenessType Type)
       : Type(Type), OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
   void printPipeline(raw_ostream &OS,
                      function_ref<StringRef(StringRef)> MapClassName2PassName);
 };
diff --git a/llvm/include/llvm/Analysis/StackSafetyAnalysis.h b/llvm/include/llvm/Analysis/StackSafetyAnalysis.h
index 751735f3e59f..2966f0c7e161 100644
--- a/llvm/include/llvm/Analysis/StackSafetyAnalysis.h
+++ b/llvm/include/llvm/Analysis/StackSafetyAnalysis.h
@@ -105,6 +105,7 @@ class StackSafetyPrinterPass : public PassInfoMixin<StackSafetyPrinterPass> {
 public:
   explicit StackSafetyPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// StackSafetyInfo wrapper for the legacy pass manager
@@ -143,6 +144,7 @@ class StackSafetyGlobalPrinterPass
 public:
   explicit StackSafetyGlobalPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// This pass performs the global (interprocedural) stack safety analysis
diff --git a/llvm/include/llvm/Analysis/StructuralHash.h b/llvm/include/llvm/Analysis/StructuralHash.h
index 0eef17d637c8..9f33c69aed34 100644
--- a/llvm/include/llvm/Analysis/StructuralHash.h
+++ b/llvm/include/llvm/Analysis/StructuralHash.h
@@ -24,6 +24,8 @@ public:
       : OS(OS), EnableDetailedStructuralHash(Detailed) {}
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+
+  static bool isRequired() { return true; }
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 048912beaba5..9697278eaeae 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1174,6 +1174,9 @@ public:
   /// \return The associativity of the cache level, if available.
   std::optional<unsigned> getCacheAssociativity(CacheLevel Level) const;
 
+  /// \return The minimum architectural page size for the target.
+  std::optional<unsigned> getMinPageSize() const;
+
   /// \return How much before a load we should place the prefetch
   /// instruction.  This is currently measured in number of
   /// instructions.
@@ -1923,6 +1926,7 @@ public:
   virtual std::optional<unsigned> getCacheSize(CacheLevel Level) const = 0;
   virtual std::optional<unsigned> getCacheAssociativity(CacheLevel Level)
       const = 0;
+  virtual std::optional<unsigned> getMinPageSize() const = 0;
 
   /// \return How much before a load we should place the prefetch
   /// instruction.  This is currently measured in number of
@@ -2520,6 +2524,10 @@ public:
     return Impl.getCacheAssociativity(Level);
   }
 
+  std::optional<unsigned> getMinPageSize() const override {
+    return Impl.getMinPageSize();
+  }
+
   /// Return the preferred prefetch distance in terms of instructions.
   ///
   unsigned getPrefetchDistance() const override {
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 7ad3ce512a35..60eab53fa2f6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -501,6 +501,8 @@ public:
     llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
   }
 
+  std::optional<unsigned> getMinPageSize() const { return {}; }
+
   unsigned getPrefetchDistance() const { return 0; }
   unsigned getMinPrefetchStride(unsigned NumMemAccesses,
                                 unsigned NumStridedMemAccesses,
@@ -1048,7 +1050,7 @@ public:
         if (TargetType->isScalableTy())
           return TTI::TCC_Basic;
         int64_t ElementSize =
-            DL.getTypeAllocSize(GTI.getIndexedType()).getFixedValue();
+            GTI.getSequentialElementStride(DL).getFixedValue();
         if (ConstIdx) {
           BaseOffset +=
               ConstIdx->getValue().sextOrTrunc(PtrSizeBits) * ElementSize;
diff --git a/llvm/include/llvm/Analysis/UniformityAnalysis.h b/llvm/include/llvm/Analysis/UniformityAnalysis.h
index f42c4950ed64..c38d100d88b8 100644
--- a/llvm/include/llvm/Analysis/UniformityAnalysis.h
+++ b/llvm/include/llvm/Analysis/UniformityAnalysis.h
@@ -47,6 +47,8 @@ public:
   explicit UniformityInfoPrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Legacy analysis pass which computes a \ref CycleInfo.
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index baa16306ebf5..7360edfce1f3 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -810,9 +810,14 @@ bool isAssumeLikeIntrinsic(const Instruction *I);
 
 /// Return true if it is valid to use the assumptions provided by an
 /// assume intrinsic, I, at the point in the control-flow identified by the
-/// context instruction, CxtI.
+/// context instruction, CxtI. By default, ephemeral values of the assumption
+/// are treated as an invalid context, to prevent the assumption from being used
+/// to optimize away its argument. If the caller can ensure that this won't
+/// happen, it can call with AllowEphemerals set to true to get more valid
+/// assumptions.
 bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI,
-                             const DominatorTree *DT = nullptr);
+                             const DominatorTree *DT = nullptr,
+                             bool AllowEphemerals = false);
 
 enum class OverflowResult {
   /// Always overflows in the direction of signed/unsigned min value.
diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index 4bffcdee6f9c..ee9207bb4f7d 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -506,12 +506,18 @@ TLI_DEFINE_VECFUNC( "llvm.log2.f64", "_ZGVnN2v_log2", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC( "log10", "_ZGVnN2v_log10", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC( "llvm.log10.f64", "_ZGVnN2v_log10", FIXED(2), "_ZGV_LLVM_N2v")
 
+TLI_DEFINE_VECFUNC( "modf", "_ZGVnN2vl8_modf", FIXED(2), "_ZGV_LLVM_N2vl8")
+
 TLI_DEFINE_VECFUNC( "pow", "_ZGVnN2vv_pow", FIXED(2), "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC( "llvm.pow.f64", "_ZGVnN2vv_pow", FIXED(2), "_ZGV_LLVM_N2vv")
 
 TLI_DEFINE_VECFUNC( "sin", "_ZGVnN2v_sin", FIXED(2), "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC( "llvm.sin.f64", "_ZGVnN2v_sin", FIXED(2), "_ZGV_LLVM_N2v")
 
+TLI_DEFINE_VECFUNC( "sincos", "_ZGVnN2vl8l8_sincos", FIXED(2), "_ZGV_LLVM_N2vl8l8")
+
+TLI_DEFINE_VECFUNC( "sincospi", "_ZGVnN2vl8l8_sincospi", FIXED(2), "_ZGV_LLVM_N2vl8l8")
+
 TLI_DEFINE_VECFUNC( "sinh", "_ZGVnN2v_sinh", FIXED(2), "_ZGV_LLVM_N2v")
 
 TLI_DEFINE_VECFUNC( "sqrt", "_ZGVnN2v_sqrt", FIXED(2), "_ZGV_LLVM_N2v")
@@ -560,12 +566,18 @@ TLI_DEFINE_VECFUNC( "llvm.log2.f32", "_ZGVnN4v_log2f", FIXED(4), "_ZGV_LLVM_N4v"
 TLI_DEFINE_VECFUNC( "log10f", "_ZGVnN4v_log10f", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC( "llvm.log10.f32", "_ZGVnN4v_log10f", FIXED(4), "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC( "modff", "_ZGVnN4vl4_modff", FIXED(4), "_ZGV_LLVM_N4vl4")
+
 TLI_DEFINE_VECFUNC( "powf", "_ZGVnN4vv_powf", FIXED(4), "_ZGV_LLVM_N4vv")
 TLI_DEFINE_VECFUNC( "llvm.pow.f32", "_ZGVnN4vv_powf", FIXED(4), "_ZGV_LLVM_N4vv")
 
 TLI_DEFINE_VECFUNC( "sinf", "_ZGVnN4v_sinf", FIXED(4), "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC( "llvm.sin.f32", "_ZGVnN4v_sinf", FIXED(4), "_ZGV_LLVM_N4v")
 
+TLI_DEFINE_VECFUNC("sincosf", "_ZGVnN4vl4l4_sincosf", FIXED(4), "_ZGV_LLVM_N4vl4l4")
+
+TLI_DEFINE_VECFUNC("sincospif", "_ZGVnN4vl4l4_sincospif", FIXED(4), "_ZGV_LLVM_N4vl4l4")
+
 TLI_DEFINE_VECFUNC( "sinhf", "_ZGVnN4v_sinhf", FIXED(4), "_ZGV_LLVM_N4v")
 
 TLI_DEFINE_VECFUNC( "sqrtf", "_ZGVnN4v_sqrtf", FIXED(4), "_ZGV_LLVM_N4v")
@@ -637,6 +649,9 @@ TLI_DEFINE_VECFUNC("log10f", "_ZGVsMxv_log10f", SCALABLE(4), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("llvm.log10.f64", "_ZGVsMxv_log10", SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("llvm.log10.f32", "_ZGVsMxv_log10f", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("modf", "_ZGVsMxvl8_modf", SCALABLE(2), MASKED, "_ZGVsMxvl8")
+TLI_DEFINE_VECFUNC("modff", "_ZGVsMxvl4_modff", SCALABLE(4), MASKED, "_ZGVsMxvl4")
+
 TLI_DEFINE_VECFUNC("pow", "_ZGVsMxvv_pow", SCALABLE(2), MASKED, "_ZGVsMxvv")
 TLI_DEFINE_VECFUNC("powf", "_ZGVsMxvv_powf", SCALABLE(4), MASKED, "_ZGVsMxvv")
 TLI_DEFINE_VECFUNC("llvm.pow.f64", "_ZGVsMxvv_pow", SCALABLE(2), MASKED, "_ZGVsMxvv")
@@ -647,6 +662,12 @@ TLI_DEFINE_VECFUNC("sinf", "_ZGVsMxv_sinf", SCALABLE(4), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("llvm.sin.f64", "_ZGVsMxv_sin", SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("llvm.sin.f32", "_ZGVsMxv_sinf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("sincos", "_ZGVsMxvl8l8_sincos", SCALABLE(2), MASKED, "_ZGVsMxvl8l8")
+TLI_DEFINE_VECFUNC("sincosf", "_ZGVsMxvl4l4_sincosf", SCALABLE(4), MASKED, "_ZGVsMxvl4l4")
+
+TLI_DEFINE_VECFUNC("sincospi", "_ZGVsMxvl8l8_sincospi", SCALABLE(2), MASKED, "_ZGVsMxvl8l8")
+TLI_DEFINE_VECFUNC("sincospif", "_ZGVsMxvl4l4_sincospif", SCALABLE(4), MASKED, "_ZGVsMxvl4l4")
+
 TLI_DEFINE_VECFUNC("sinh", "_ZGVsMxv_sinh",  SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("sinhf", "_ZGVsMxv_sinhf", SCALABLE(4), MASKED, "_ZGVsMxv")
 
@@ -834,6 +855,11 @@ TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_vlog10q_f32", FIXED(4), NOMASK, "_ZG
 TLI_DEFINE_VECFUNC("llvm.log10.f64", "armpl_svlog10_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_svlog10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("modf", "armpl_vmodfq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8")
+TLI_DEFINE_VECFUNC("modff", "armpl_vmodfq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4")
+TLI_DEFINE_VECFUNC("modf", "armpl_svmodf_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvl8")
+TLI_DEFINE_VECFUNC("modff", "armpl_svmodf_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4")
+
 TLI_DEFINE_VECFUNC("nextafter", "armpl_vnextafterq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
 TLI_DEFINE_VECFUNC("nextafterf", "armpl_vnextafterq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
 TLI_DEFINE_VECFUNC("nextafter", "armpl_svnextafter_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
@@ -859,6 +885,16 @@ TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_vsinq_f32", FIXED(4), NOMASK, "_ZGV_LL
 TLI_DEFINE_VECFUNC("llvm.sin.f64", "armpl_svsin_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
 TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_svsin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
 
+TLI_DEFINE_VECFUNC("sincos", "armpl_vsincosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8")
+TLI_DEFINE_VECFUNC("sincosf", "armpl_vsincosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4")
+TLI_DEFINE_VECFUNC("sincos", "armpl_svsincos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvl8l8")
+TLI_DEFINE_VECFUNC("sincosf", "armpl_svsincos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4")
+
+TLI_DEFINE_VECFUNC("sincospi", "armpl_vsincospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8")
+TLI_DEFINE_VECFUNC("sincospif", "armpl_vsincospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4")
+TLI_DEFINE_VECFUNC("sincospi", "armpl_svsincospi_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8")
+TLI_DEFINE_VECFUNC("sincospif", "armpl_svsincospi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4")
+
 TLI_DEFINE_VECFUNC("sinh", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
 TLI_DEFINE_VECFUNC("sinhf", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
 TLI_DEFINE_VECFUNC("sinh", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 0f968eac36e7..9b8128a9ec40 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -356,6 +356,7 @@ enum {
   ELFOSABI_AROS = 15,          // AROS
   ELFOSABI_FENIXOS = 16,       // FenixOS
   ELFOSABI_CLOUDABI = 17,      // Nuxi CloudABI
+  ELFOSABI_CUDA = 51,          // NVIDIA CUDA architecture.
   ELFOSABI_FIRST_ARCH = 64,    // First architecture-specific OS ABI
   ELFOSABI_AMDGPU_HSA = 64,    // AMD HSA runtime
   ELFOSABI_AMDGPU_PAL = 65,    // AMD PAL runtime
diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def
index df3a342151fb..4859057abcbb 100644
--- a/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def
+++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/LoongArch.def
@@ -126,3 +126,26 @@ ELF_RELOC(R_LARCH_64_PCREL,    109)
 //
 // Spec addition: https://github.com/loongson/la-abi-specs/pull/4
 ELF_RELOC(R_LARCH_CALL36, 110)
+
+// Relocs added in ELF for the LoongArch™ Architecture v20231219, part of the
+// v2.30 LoongArch ABI specs.
+//
+// Spec addition: https://github.com/loongson/la-abi-specs/pull/5
+ELF_RELOC(R_LARCH_TLS_DESC32,           13)
+ELF_RELOC(R_LARCH_TLS_DESC64,           14)
+ELF_RELOC(R_LARCH_TLS_DESC_PC_HI20,    111)
+ELF_RELOC(R_LARCH_TLS_DESC_PC_LO12,    112)
+ELF_RELOC(R_LARCH_TLS_DESC64_PC_LO20,  113)
+ELF_RELOC(R_LARCH_TLS_DESC64_PC_HI12,  114)
+ELF_RELOC(R_LARCH_TLS_DESC_HI20,       115)
+ELF_RELOC(R_LARCH_TLS_DESC_LO12,       116)
+ELF_RELOC(R_LARCH_TLS_DESC64_LO20,     117)
+ELF_RELOC(R_LARCH_TLS_DESC64_HI12,     118)
+ELF_RELOC(R_LARCH_TLS_DESC_LD,         119)
+ELF_RELOC(R_LARCH_TLS_DESC_CALL,       120)
+ELF_RELOC(R_LARCH_TLS_LE_HI20_R,       121)
+ELF_RELOC(R_LARCH_TLS_LE_ADD_R,        122)
+ELF_RELOC(R_LARCH_TLS_LE_LO12_R,       123)
+ELF_RELOC(R_LARCH_TLS_LD_PCREL20_S2,   124)
+ELF_RELOC(R_LARCH_TLS_GD_PCREL20_S2,   125)
+ELF_RELOC(R_LARCH_TLS_DESC_PCREL20_S2, 126)
diff --git a/llvm/include/llvm/Bitstream/BitstreamWriter.h b/llvm/include/llvm/Bitstream/BitstreamWriter.h
index f7d362b5d70c..c726508cd528 100644
--- a/llvm/include/llvm/Bitstream/BitstreamWriter.h
+++ b/llvm/include/llvm/Bitstream/BitstreamWriter.h
@@ -239,7 +239,8 @@ public:
 
     // Emit the bits with VBR encoding, NumBits-1 bits at a time.
     while (Val >= Threshold) {
-      Emit((Val & ((1 << (NumBits-1))-1)) | (1 << (NumBits-1)), NumBits);
+      Emit((Val & ((1U << (NumBits - 1)) - 1)) | (1U << (NumBits - 1)),
+           NumBits);
       Val >>= NumBits-1;
     }
 
@@ -255,7 +256,8 @@ public:
 
     // Emit the bits with VBR encoding, NumBits-1 bits at a time.
     while (Val >= Threshold) {
-      Emit(((uint32_t)Val & ((1 << (NumBits - 1)) - 1)) | (1 << (NumBits - 1)),
+      Emit(((uint32_t)Val & ((1U << (NumBits - 1)) - 1)) |
+               (1U << (NumBits - 1)),
            NumBits);
       Val >>= NumBits-1;
     }
diff --git a/llvm/include/llvm/CodeGen/AccelTable.h b/llvm/include/llvm/CodeGen/AccelTable.h
index 6eb09f32f9f9..0638fbffda4f 100644
--- a/llvm/include/llvm/CodeGen/AccelTable.h
+++ b/llvm/include/llvm/CodeGen/AccelTable.h
@@ -143,6 +143,15 @@ public:
     std::vector<AccelTableData *> Values;
     MCSymbol *Sym;
 
+    /// Get all AccelTableData cast as a `T`.
+    template <typename T = AccelTableData *> auto getValues() const {
+      static_assert(std::is_pointer<T>());
+      static_assert(
+          std::is_base_of<AccelTableData, std::remove_pointer_t<T>>());
+      return map_range(
+          Values, [](AccelTableData *Data) { return static_cast<T>(Data); });
+    }
+
 #ifndef NDEBUG
     void print(raw_ostream &OS) const;
     void dump() const { print(dbgs()); }
@@ -319,8 +328,7 @@ public:
   /// Needs to be called after DIE offsets are computed.
   void convertDieToOffset() {
     for (auto &Entry : Entries) {
-      for (AccelTableData *Value : Entry.second.Values) {
-        DWARF5AccelTableData *Data = static_cast<DWARF5AccelTableData *>(Value);
+      for (auto *Data : Entry.second.getValues<DWARF5AccelTableData *>()) {
         // For TU we normalize as each Unit is emitted.
         // So when this is invoked after CU construction we will be in mixed
         // state.
@@ -332,8 +340,7 @@ public:
 
   void addTypeEntries(DWARF5AccelTable &Table) {
     for (auto &Entry : Table.getEntries()) {
-      for (AccelTableData *Value : Entry.second.Values) {
-        DWARF5AccelTableData *Data = static_cast<DWARF5AccelTableData *>(Value);
+      for (auto *Data : Entry.second.getValues<DWARF5AccelTableData *>()) {
         addName(Entry.second.Name, Data->getDieOffset(), Data->getDieTag(),
                 Data->getUnitID(), true);
       }
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index dfb8d5d9f2f5..bba675f1d3eb 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -21,11 +21,14 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/LineIterator.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Target/TargetMachine.h"
+
 using namespace llvm;
 
 namespace llvm {
@@ -72,25 +75,13 @@ template <> struct DenseMapInfo<UniqueBBID> {
   }
 };
 
-class BasicBlockSectionsProfileReader : public ImmutablePass {
+class BasicBlockSectionsProfileReader {
 public:
-  static char ID;
-
+  friend class BasicBlockSectionsProfileReaderWrapperPass;
   BasicBlockSectionsProfileReader(const MemoryBuffer *Buf)
-      : ImmutablePass(ID), MBuf(Buf),
-        LineIt(*Buf, /*SkipBlanks=*/true, /*CommentMarker=*/'#') {
-    initializeBasicBlockSectionsProfileReaderPass(
-        *PassRegistry::getPassRegistry());
-  };
+      : MBuf(Buf), LineIt(*Buf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'){};
 
-  BasicBlockSectionsProfileReader() : ImmutablePass(ID) {
-    initializeBasicBlockSectionsProfileReaderPass(
-        *PassRegistry::getPassRegistry());
-  }
-
-  StringRef getPassName() const override {
-    return "Basic Block Sections Profile Reader";
-  }
+  BasicBlockSectionsProfileReader(){};
 
   // Returns true if basic block sections profile exist for function \p
   // FuncName.
@@ -109,10 +100,6 @@ public:
   SmallVector<SmallVector<unsigned>>
   getClonePathsForFunction(StringRef FuncName) const;
 
-  // Initializes the FunctionNameToDIFilename map for the current module and
-  // then reads the profile for the matching functions.
-  bool doInitialization(Module &M) override;
-
 private:
   StringRef getAliasName(StringRef FuncName) const {
     auto R = FuncAliasMap.find(FuncName);
@@ -170,7 +157,61 @@ private:
 // sections profile. \p Buf is a memory buffer that contains the list of
 // functions and basic block ids to selectively enable basic block sections.
 ImmutablePass *
-createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf);
+createBasicBlockSectionsProfileReaderWrapperPass(const MemoryBuffer *Buf);
+
+/// Analysis pass providing the \c BasicBlockSectionsProfileReader.
+///
+/// Note that this pass's result cannot be invalidated, it is immutable for the
+/// life of the module.
+class BasicBlockSectionsProfileReaderAnalysis
+    : public AnalysisInfoMixin<BasicBlockSectionsProfileReaderAnalysis> {
+
+public:
+  static AnalysisKey Key;
+  typedef BasicBlockSectionsProfileReader Result;
+  BasicBlockSectionsProfileReaderAnalysis(const TargetMachine *TM) : TM(TM) {}
+
+  Result run(Function &F, FunctionAnalysisManager &AM);
+
+private:
+  const TargetMachine *TM;
+};
+
+class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass {
+public:
+  static char ID;
+  BasicBlockSectionsProfileReader BBSPR;
+
+  BasicBlockSectionsProfileReaderWrapperPass(const MemoryBuffer *Buf)
+      : ImmutablePass(ID), BBSPR(BasicBlockSectionsProfileReader(Buf)) {
+    initializeBasicBlockSectionsProfileReaderWrapperPassPass(
+        *PassRegistry::getPassRegistry());
+  };
+
+  BasicBlockSectionsProfileReaderWrapperPass()
+      : ImmutablePass(ID), BBSPR(BasicBlockSectionsProfileReader()) {
+    initializeBasicBlockSectionsProfileReaderWrapperPassPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  StringRef getPassName() const override {
+    return "Basic Block Sections Profile Reader";
+  }
+
+  bool isFunctionHot(StringRef FuncName) const;
+
+  std::pair<bool, SmallVector<BBClusterInfo>>
+  getClusterInfoForFunction(StringRef FuncName) const;
+
+  SmallVector<SmallVector<unsigned>>
+  getClonePathsForFunction(StringRef FuncName) const;
+
+  // Initializes the FunctionNameToDIFilename map for the current module and
+  // then reads the profile for the matching functions.
+  bool doInitialization(Module &M) override;
+
+  BasicBlockSectionsProfileReader &getBBSPR();
+};
 
 } // namespace llvm
 #endif // LLVM_CODEGEN_BASICBLOCKSECTIONSPROFILEREADER_H
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index 06da9fb57902..fa81ff504ac6 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -24,6 +24,7 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
 #include "llvm/CodeGen/CallBrPrepare.h"
+#include "llvm/CodeGen/CodeGenPrepare.h"
 #include "llvm/CodeGen/DwarfEHPrepare.h"
 #include "llvm/CodeGen/ExpandMemCmp.h"
 #include "llvm/CodeGen/ExpandReductions.h"
@@ -38,7 +39,9 @@
 #include "llvm/CodeGen/ReplaceWithVeclib.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
+#include "llvm/CodeGen/ShadowStackGCLowering.h"
 #include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/UnreachableBlockElim.h"
 #include "llvm/CodeGen/WasmEHPrepare.h"
 #include "llvm/CodeGen/WinEHPrepare.h"
@@ -229,25 +232,27 @@ protected:
         C(&PassT::Key);
     }
 
-    template <typename PassT> void insertPass(AnalysisKey *ID, PassT Pass) {
+    template <typename PassT> void insertPass(MachinePassKey *ID, PassT Pass) {
       AfterCallbacks.emplace_back(
-          [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) {
+          [this, ID, Pass = std::move(Pass)](MachinePassKey *PassID) {
             if (PassID == ID)
               this->PM.addPass(std::move(Pass));
           });
     }
 
-    void disablePass(AnalysisKey *ID) {
+    void disablePass(MachinePassKey *ID) {
       BeforeCallbacks.emplace_back(
-          [ID](AnalysisKey *PassID) { return PassID != ID; });
+          [ID](MachinePassKey *PassID) { return PassID != ID; });
     }
 
     MachineFunctionPassManager releasePM() { return std::move(PM); }
 
   private:
     MachineFunctionPassManager &PM;
-    SmallVector<llvm::unique_function<bool(AnalysisKey *)>, 4> BeforeCallbacks;
-    SmallVector<llvm::unique_function<void(AnalysisKey *)>, 4> AfterCallbacks;
+    SmallVector<llvm::unique_function<bool(MachinePassKey *)>, 4>
+        BeforeCallbacks;
+    SmallVector<llvm::unique_function<void(MachinePassKey *)>, 4>
+        AfterCallbacks;
   };
 
   LLVMTargetMachine &TM;
@@ -485,6 +490,7 @@ Error CodeGenPassBuilder<Derived>::buildPipeline(
   AddIRPass addIRPass(MPM, Opt.DebugPM);
   // `ProfileSummaryInfo` is always valid.
   addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+  addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
   addISelPasses(addIRPass);
 
   AddMachinePass addPass(MFPM);
@@ -640,6 +646,9 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
   // Run GC lowering passes for builtin collectors
   // TODO: add a pass insertion point here
   addPass(GCLoweringPass());
+  // FIXME: `ShadowStackGCLoweringPass` now is a
+  // module pass, so it will trigger assertion.
+  // See comment of `AddingFunctionPasses`
   addPass(ShadowStackGCLoweringPass());
   addPass(LowerConstantIntrinsicsPass());
 
@@ -727,7 +736,7 @@ void CodeGenPassBuilder<Derived>::addPassesToHandleExceptions(
 template <typename Derived>
 void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
   if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
-    addPass(CodeGenPreparePass());
+    addPass(CodeGenPreparePass(&TM));
   // TODO: Default ctor'd RewriteSymbolPass is no-op.
   // addPass(RewriteSymbolPass());
 }
@@ -742,7 +751,7 @@ void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
   // Add both the safe stack and the stack protection passes: each of them will
   // only protect functions that have corresponding attributes.
   addPass(SafeStackPass(&TM));
-  addPass(StackProtectorPass());
+  addPass(StackProtectorPass(&TM));
 
   if (Opt.PrintISelInput)
     addPass(PrintFunctionPass(dbgs(),
diff --git a/llvm/include/llvm/CodeGen/CodeGenPrepare.h b/llvm/include/llvm/CodeGen/CodeGenPrepare.h
new file mode 100644
index 000000000000..dee3a9ee53d7
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/CodeGenPrepare.h
@@ -0,0 +1,35 @@
+//===- CodeGenPrepare.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Defines an IR pass for CodeGen Prepare.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_PREPARE_H
+#define LLVM_CODEGEN_PREPARE_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Function;
+class TargetMachine;
+
+class CodeGenPreparePass : public PassInfoMixin<CodeGenPreparePass> {
+private:
+  const TargetMachine *TM;
+
+public:
+  CodeGenPreparePass(const TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_PREPARE_H
diff --git a/llvm/include/llvm/CodeGen/FastISel.h b/llvm/include/llvm/CodeGen/FastISel.h
index dc2931b40d35..0f17e51f0b7a 100644
--- a/llvm/include/llvm/CodeGen/FastISel.h
+++ b/llvm/include/llvm/CodeGen/FastISel.h
@@ -319,6 +319,10 @@ public:
   /// Reset InsertPt to the given old insert position.
   void leaveLocalValueArea(SavePoint Old);
 
+  /// Target-independent lowering of non-instruction debug info associated with
+  /// this instruction.
+  void handleDbgInfo(const Instruction *II);
+
 protected:
   explicit FastISel(FunctionLoweringInfo &FuncInfo,
                     const TargetLibraryInfo *LibInfo,
@@ -518,6 +522,16 @@ protected:
     return MF->getFunction().hasOptSize();
   }
 
+  /// Target-independent lowering of debug information. Returns false if the
+  /// debug information couldn't be lowered and was instead discarded.
+  virtual bool lowerDbgValue(const Value *V, DIExpression *Expr,
+                             DILocalVariable *Var, const DebugLoc &DL);
+
+  /// Target-independent lowering of debug information. Returns false if the
+  /// debug information couldn't be lowered and was instead discarded.
+  virtual bool lowerDbgDeclare(const Value *V, DIExpression *Expr,
+                               DILocalVariable *Var, const DebugLoc &DL);
+
 private:
   /// Handle PHI nodes in successor blocks.
   ///
diff --git a/llvm/include/llvm/CodeGen/GCMetadata.h b/llvm/include/llvm/CodeGen/GCMetadata.h
index 9e4e8342ea29..ca6a511185c7 100644
--- a/llvm/include/llvm/CodeGen/GCMetadata.h
+++ b/llvm/include/llvm/CodeGen/GCMetadata.h
@@ -186,6 +186,17 @@ public:
   Result run(Function &F, FunctionAnalysisManager &FAM);
 };
 
+/// LowerIntrinsics - This pass rewrites calls to the llvm.gcread or
+/// llvm.gcwrite intrinsics, replacing them with simple loads and stores as
+/// directed by the GCStrategy. It also performs automatic root initialization
+/// and custom intrinsic lowering.
+///
+/// This pass requires `CollectorMetadataAnalysis`.
+class GCLoweringPass : public PassInfoMixin<GCLoweringPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
 /// An analysis pass which caches information about the entire Module.
 /// Records both the function level information used by GCRoots and a
 /// cache of the 'active' gc strategy objects for the current Module.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index bffc03ed0187..1b094d9d9fe7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -366,6 +366,10 @@ private:
                        BranchProbability BranchProbToNext, Register Reg,
                        SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB);
 
+  void splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
+                     const SwitchCG::SwitchWorkListItem &W, Value *Cond,
+                     MachineBasicBlock *SwitchMBB, MachineIRBuilder &MIB);
+
   bool lowerJumpTableWorkItem(
       SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
       MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 851353042cc2..da330b517c28 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -1366,6 +1366,9 @@ public:
           // Adding Use to ArtifactList.
           WrapperObserver.changedInstr(Use);
           break;
+        case TargetOpcode::G_ASSERT_SEXT:
+        case TargetOpcode::G_ASSERT_ZEXT:
+        case TargetOpcode::G_ASSERT_ALIGN:
         case TargetOpcode::COPY: {
           Register Copy = Use.getOperand(0).getReg();
           if (Copy.isVirtual())
@@ -1392,6 +1395,9 @@ private:
     case TargetOpcode::G_ANYEXT:
     case TargetOpcode::G_SEXT:
     case TargetOpcode::G_EXTRACT:
+    case TargetOpcode::G_ASSERT_SEXT:
+    case TargetOpcode::G_ASSERT_ZEXT:
+    case TargetOpcode::G_ASSERT_ALIGN:
       return MI.getOperand(1).getReg();
     case TargetOpcode::G_UNMERGE_VALUES:
       return MI.getOperand(MI.getNumOperands() - 1).getReg();
@@ -1425,7 +1431,8 @@ private:
       if (MRI.hasOneUse(PrevRegSrc)) {
         if (TmpDef != &DefMI) {
           assert((TmpDef->getOpcode() == TargetOpcode::COPY ||
-                  isArtifactCast(TmpDef->getOpcode())) &&
+                  isArtifactCast(TmpDef->getOpcode()) ||
+                  isPreISelGenericOptimizationHint(TmpDef->getOpcode())) &&
                  "Expecting copy or artifact cast here");
 
           DeadInsts.push_back(TmpDef);
@@ -1509,16 +1516,8 @@ private:
   /// Looks through copy instructions and returns the actual
   /// source register.
   Register lookThroughCopyInstrs(Register Reg) {
-    using namespace llvm::MIPatternMatch;
-
-    Register TmpReg;
-    while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) {
-      if (MRI.getType(TmpReg).isValid())
-        Reg = TmpReg;
-      else
-        break;
-    }
-    return Reg;
+    Register TmpReg = getSrcRegIgnoringCopies(Reg, MRI);
+    return TmpReg.isValid() ? TmpReg : Reg;
   }
 };
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 711ba10247c3..586679fa2954 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -288,11 +288,14 @@ private:
 
   // Implements floating-point environment read/write via library function call.
   LegalizeResult createGetStateLibcall(MachineIRBuilder &MIRBuilder,
-                                       MachineInstr &MI);
+                                       MachineInstr &MI,
+                                       LostDebugLocObserver &LocObserver);
   LegalizeResult createSetStateLibcall(MachineIRBuilder &MIRBuilder,
-                                       MachineInstr &MI);
+                                       MachineInstr &MI,
+                                       LostDebugLocObserver &LocObserver);
   LegalizeResult createResetStateLibcall(MachineIRBuilder &MIRBuilder,
-                                         MachineInstr &MI);
+                                         MachineInstr &MI,
+                                         LostDebugLocObserver &LocObserver);
 
 public:
   /// Return the alignment to use for a stack temporary object with the given
@@ -347,6 +350,9 @@ public:
 
   LegalizeResult fewerElementsVectorReductions(MachineInstr &MI,
                                                unsigned TypeIdx, LLT NarrowTy);
+  LegalizeResult fewerElementsVectorSeqReductions(MachineInstr &MI,
+                                                  unsigned TypeIdx,
+                                                  LLT NarrowTy);
 
   LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx,
                                             LLT NarrowTy);
@@ -440,13 +446,15 @@ public:
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
               const CallLowering::ArgInfo &Result,
-              ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC);
+              ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC,
+              LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr);
 
 /// Helper function that creates the given libcall.
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
               const CallLowering::ArgInfo &Result,
-              ArrayRef<CallLowering::ArgInfo> Args);
+              ArrayRef<CallLowering::ArgInfo> Args,
+              LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr);
 
 /// Create a libcall to memcpy et al.
 LegalizerHelper::LegalizeResult
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index e51a3ec94005..d09100b28f3e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -35,6 +35,7 @@ extern cl::opt<bool> DisableGISelLegalityCheck;
 class MachineFunction;
 class raw_ostream;
 class LegalizerHelper;
+class LostDebugLocObserver;
 class MachineInstr;
 class MachineRegisterInfo;
 class MCInstrInfo;
@@ -223,6 +224,11 @@ struct TypePairAndMemDesc {
   }
 };
 
+/// True iff P is false.
+template <typename Predicate> Predicate predNot(Predicate P) {
+  return [=](const LegalityQuery &Query) { return !P(Query); };
+}
+
 /// True iff P0 and P1 are true.
 template<typename Predicate>
 Predicate all(Predicate P0, Predicate P1) {
@@ -1288,8 +1294,8 @@ public:
                        const MachineRegisterInfo &MRI) const;
 
   /// Called for instructions with the Custom LegalizationAction.
-  virtual bool legalizeCustom(LegalizerHelper &Helper,
-                              MachineInstr &MI) const {
+  virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
+                              LostDebugLocObserver &LocObserver) const {
     llvm_unreachable("must implement this if custom action is used");
   }
 
diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
index f950dfae7e33..e789747036ef 100644
--- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def
+++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
@@ -26,6 +26,7 @@ MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
 MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ())
 MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass, ())
 MODULE_PASS("lower-emutls", LowerEmuTLSPass, ())
+MODULE_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
 #undef MODULE_PASS
 
 #ifndef FUNCTION_ANALYSIS
@@ -33,6 +34,7 @@ MODULE_PASS("lower-emutls", LowerEmuTLSPass, ())
 #endif
 FUNCTION_ANALYSIS("gc-function", GCFunctionAnalysis, ())
 FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
+FUNCTION_ANALYSIS("ssp-layout", SSPLayoutAnalysis, ())
 FUNCTION_ANALYSIS("targetir", TargetIRAnalysis,
                   (std::move(TM.getTargetIRAnalysis())))
 #undef FUNCTION_ANALYSIS
@@ -42,6 +44,7 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis,
 #endif
 FUNCTION_PASS("callbrprepare", CallBrPreparePass, ())
 FUNCTION_PASS("cfguard", CFGuardPass, ())
+FUNCTION_PASS("codegenprepare", CodeGenPreparePass, (TM))
 FUNCTION_PASS("consthoist", ConstantHoistingPass, ())
 FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM))
 FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false))
@@ -50,6 +53,7 @@ FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, (TM))
 FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass, (TM))
 FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ())
 FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ())
+FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
 FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, (TM))
 FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM))
 FUNCTION_PASS("interleaved-load-combine", InterleavedLoadCombinePass, (TM))
@@ -63,6 +67,7 @@ FUNCTION_PASS("safe-stack", SafeStackPass, (TM))
 FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass, ())
 FUNCTION_PASS("select-optimize", SelectOptimizePass, (TM))
 FUNCTION_PASS("sjlj-eh-prepare", SjLjEHPreparePass, (TM))
+FUNCTION_PASS("stack-protector", StackProtectorPass, (TM))
 FUNCTION_PASS("tlshoist", TLSVariableHoistPass, ())
 FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass, ())
 FUNCTION_PASS("verify", VerifierPass, ())
@@ -131,10 +136,6 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis,
 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
 #endif
 DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ())
-DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ())
-DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
-DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
-DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ())
 #undef DUMMY_FUNCTION_PASS
 
 #ifndef DUMMY_MACHINE_MODULE_PASS
diff --git a/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h b/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h
index fe07c70d85c5..3dc0731f5a04 100644
--- a/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h
+++ b/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h
@@ -32,7 +32,7 @@ public:
       bool PutEmptyString = false)
       : Translator(Translator) {
     if (PutEmptyString)
-      EmptyString = getEntry("");
+      getEntry("");
   }
 
   DwarfStringPoolEntryRef getEntry(StringRef S);
@@ -59,7 +59,6 @@ private:
   MapTy Strings;
   uint64_t CurrentEndOffset = 0;
   unsigned NumEntries = 0;
-  DwarfStringPoolEntryRef EmptyString;
   std::function<StringRef(StringRef Input)> Translator;
 };
 
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index ca9fbb1def76..bbfb8a0dbe26 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -93,9 +93,9 @@ namespace llvm {
   MachineFunctionPass *createResetMachineFunctionPass(bool EmitFallbackDiag,
                                                       bool AbortOnFailedISel);
 
-  /// createCodeGenPreparePass - Transform the code to expose more pattern
+  /// createCodeGenPrepareLegacyPass - Transform the code to expose more pattern
   /// matching during instruction selection.
-  FunctionPass *createCodeGenPreparePass();
+  FunctionPass *createCodeGenPrepareLegacyPass();
 
   /// This pass implements generation of target-specific intrinsics to support
   /// handling of complex number arithmetic
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
index 666420681510..3a407c4a4d94 100644
--- a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -82,6 +82,12 @@ namespace RTLIB {
   /// UNKNOWN_LIBCALL if there is none.
   Libcall getSYNC(unsigned Opc, MVT VT);
 
+  /// Return the outline atomics value for the given atomic ordering, access
+  /// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
+  /// is none.
+  Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4],
+                                 AtomicOrdering Order, uint64_t MemSize);
+
   /// Return the outline atomics value for the given opcode, atomic ordering
   /// and type, or UNKNOWN_LIBCALL if there is none.
   Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 5c44538fe699..ebf410cc94de 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -381,6 +381,7 @@ private:
   bool NoUnsignedWrap : 1;
   bool NoSignedWrap : 1;
   bool Exact : 1;
+  bool Disjoint : 1;
   bool NonNeg : 1;
   bool NoNaNs : 1;
   bool NoInfs : 1;
@@ -402,10 +403,11 @@ private:
 public:
   /// Default constructor turns off all optimization flags.
   SDNodeFlags()
-      : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NonNeg(false),
-        NoNaNs(false), NoInfs(false), NoSignedZeros(false),
-        AllowReciprocal(false), AllowContract(false), ApproximateFuncs(false),
-        AllowReassociation(false), NoFPExcept(false), Unpredictable(false) {}
+      : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false),
+        Disjoint(false), NonNeg(false), NoNaNs(false), NoInfs(false),
+        NoSignedZeros(false), AllowReciprocal(false), AllowContract(false),
+        ApproximateFuncs(false), AllowReassociation(false), NoFPExcept(false),
+        Unpredictable(false) {}
 
   /// Propagate the fast-math-flags from an IR FPMathOperator.
   void copyFMF(const FPMathOperator &FPMO) {
@@ -422,6 +424,7 @@ public:
   void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
   void setNoSignedWrap(bool b) { NoSignedWrap = b; }
   void setExact(bool b) { Exact = b; }
+  void setDisjoint(bool b) { Disjoint = b; }
   void setNonNeg(bool b) { NonNeg = b; }
   void setNoNaNs(bool b) { NoNaNs = b; }
   void setNoInfs(bool b) { NoInfs = b; }
@@ -437,6 +440,7 @@ public:
   bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
   bool hasNoSignedWrap() const { return NoSignedWrap; }
   bool hasExact() const { return Exact; }
+  bool hasDisjoint() const { return Disjoint; }
   bool hasNonNeg() const { return NonNeg; }
   bool hasNoNaNs() const { return NoNaNs; }
   bool hasNoInfs() const { return NoInfs; }
@@ -454,6 +458,7 @@ public:
     NoUnsignedWrap &= Flags.NoUnsignedWrap;
     NoSignedWrap &= Flags.NoSignedWrap;
     Exact &= Flags.Exact;
+    Disjoint &= Flags.Disjoint;
     NonNeg &= Flags.NonNeg;
     NoNaNs &= Flags.NoNaNs;
     NoInfs &= Flags.NoInfs;
@@ -924,6 +929,9 @@ public:
   /// Helper method returns the integer value of a ConstantSDNode operand.
   inline uint64_t getConstantOperandVal(unsigned Num) const;
 
+  /// Helper method returns the zero-extended integer value of a ConstantSDNode.
+  inline uint64_t getAsZExtVal() const;
+
   /// Helper method returns the APInt of a ConstantSDNode operand.
   inline const APInt &getConstantOperandAPInt(unsigned Num) const;
 
@@ -1640,6 +1648,10 @@ uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
   return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
 }
 
+uint64_t SDNode::getAsZExtVal() const {
+  return cast<ConstantSDNode>(this)->getZExtValue();
+}
+
 const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
   return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
 }
diff --git a/llvm/include/llvm/CodeGen/ShadowStackGCLowering.h b/llvm/include/llvm/CodeGen/ShadowStackGCLowering.h
new file mode 100644
index 000000000000..1586c6cf545b
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/ShadowStackGCLowering.h
@@ -0,0 +1,24 @@
+//===- llvm/CodeGen/ShadowStackGCLowering.h ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_SHADOWSTACKGCLOWERING_H
+#define LLVM_CODEGEN_SHADOWSTACKGCLOWERING_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class ShadowStackGCLoweringPass
+    : public PassInfoMixin<ShadowStackGCLoweringPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_SHADOWSTACKGCLOWERING_H
diff --git a/llvm/include/llvm/CodeGen/StackProtector.h b/llvm/include/llvm/CodeGen/StackProtector.h
index 57cb7a1c85ae..eb5d9d0caebc 100644
--- a/llvm/include/llvm/CodeGen/StackProtector.h
+++ b/llvm/include/llvm/CodeGen/StackProtector.h
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -30,25 +31,15 @@ class Module;
 class TargetLoweringBase;
 class TargetMachine;
 
-class StackProtector : public FunctionPass {
-private:
+class SSPLayoutInfo {
+  friend class StackProtectorPass;
+  friend class SSPLayoutAnalysis;
+  friend class StackProtector;
   static constexpr unsigned DefaultSSPBufferSize = 8;
 
   /// A mapping of AllocaInsts to their required SSP layout.
-  using SSPLayoutMap = DenseMap<const AllocaInst *,
-                                MachineFrameInfo::SSPLayoutKind>;
-
-  const TargetMachine *TM = nullptr;
-
-  /// TLI - Keep a pointer of a TargetLowering to consult for determining
-  /// target type sizes.
-  const TargetLoweringBase *TLI = nullptr;
-  Triple Trip;
-
-  Function *F = nullptr;
-  Module *M = nullptr;
-
-  std::optional<DomTreeUpdater> DTU;
+  using SSPLayoutMap =
+      DenseMap<const AllocaInst *, MachineFrameInfo::SSPLayoutKind>;
 
   /// Layout - Mapping of allocations to the required SSPLayoutKind.
   /// StackProtector analysis will update this map when determining if an
@@ -59,23 +50,59 @@ private:
   /// protection when -fstack-protection is used.
   unsigned SSPBufferSize = DefaultSSPBufferSize;
 
+  bool RequireStackProtector = false;
+
   // A prologue is generated.
   bool HasPrologue = false;
 
   // IR checking code is generated.
   bool HasIRCheck = false;
 
-  /// InsertStackProtectors - Insert code into the prologue and epilogue of
-  /// the function.
-  ///
-  ///  - The prologue code loads and stores the stack guard onto the stack.
-  ///  - The epilogue checks the value stored in the prologue against the
-  ///    original value. It calls __stack_chk_fail if they differ.
-  bool InsertStackProtectors();
+public:
+  // Return true if StackProtector is supposed to be handled by SelectionDAG.
+  bool shouldEmitSDCheck(const BasicBlock &BB) const;
+
+  void copyToMachineFrameInfo(MachineFrameInfo &MFI) const;
+};
+
+class SSPLayoutAnalysis : public AnalysisInfoMixin<SSPLayoutAnalysis> {
+  friend AnalysisInfoMixin<SSPLayoutAnalysis>;
+  using SSPLayoutMap = SSPLayoutInfo::SSPLayoutMap;
+
+  static AnalysisKey Key;
+
+public:
+  using Result = SSPLayoutInfo;
+
+  Result run(Function &F, FunctionAnalysisManager &FAM);
 
-  /// CreateFailBB - Create a basic block to jump to when the stack protector
-  /// check fails.
-  BasicBlock *CreateFailBB();
+  /// Check whether or not \p F needs a stack protector based upon the stack
+  /// protector level.
+  static bool requiresStackProtector(Function *F,
+                                     SSPLayoutMap *Layout = nullptr);
+};
+
+class StackProtectorPass : public PassInfoMixin<StackProtectorPass> {
+  const TargetMachine *TM;
+
+public:
+  explicit StackProtectorPass(const TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+class StackProtector : public FunctionPass {
+private:
+  /// A mapping of AllocaInsts to their required SSP layout.
+  using SSPLayoutMap = SSPLayoutInfo::SSPLayoutMap;
+
+  const TargetMachine *TM = nullptr;
+
+  Function *F = nullptr;
+  Module *M = nullptr;
+
+  std::optional<DomTreeUpdater> DTU;
+
+  SSPLayoutInfo LayoutInfo;
 
 public:
   static char ID; // Pass identification, replacement for typeid.
@@ -85,16 +112,22 @@ public:
   void getAnalysisUsage(AnalysisUsage &AU) const override;
 
   // Return true if StackProtector is supposed to be handled by SelectionDAG.
-  bool shouldEmitSDCheck(const BasicBlock &BB) const;
+  bool shouldEmitSDCheck(const BasicBlock &BB) const {
+    return LayoutInfo.shouldEmitSDCheck(BB);
+  }
 
   bool runOnFunction(Function &Fn) override;
 
-  void copyToMachineFrameInfo(MachineFrameInfo &MFI) const;
+  void copyToMachineFrameInfo(MachineFrameInfo &MFI) const {
+    LayoutInfo.copyToMachineFrameInfo(MFI);
+  }
 
   /// Check whether or not \p F needs a stack protector based upon the stack
   /// protector level.
-  static bool requiresStackProtector(Function *F, SSPLayoutMap *Layout = nullptr);
-
+  static bool requiresStackProtector(Function *F,
+                                     SSPLayoutMap *Layout = nullptr) {
+    return SSPLayoutAnalysis::requiresStackProtector(F, Layout);
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
index 5d06e21737b8..99478e9f39e2 100644
--- a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
+++ b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
@@ -293,6 +293,22 @@ public:
       MachineBasicBlock *Src, MachineBasicBlock *Dst,
       BranchProbability Prob = BranchProbability::getUnknown()) = 0;
 
+  /// Determine the rank by weight of CC in [First,Last]. If CC has more weight
+  /// than each cluster in the range, its rank is 0.
+  unsigned caseClusterRank(const CaseCluster &CC, CaseClusterIt First,
+                           CaseClusterIt Last);
+
+  struct SplitWorkItemInfo {
+    CaseClusterIt LastLeft;
+    CaseClusterIt FirstRight;
+    BranchProbability LeftProb;
+    BranchProbability RightProb;
+  };
+  /// Compute information to balance the tree based on branch probabilities to
+  /// create a near-optimal (in terms of search time given key frequency) binary
+  /// search tree. See e.g. Kurt Mehlhorn "Nearly Optimal Binary Search Trees"
+  /// (1975).
+  SplitWorkItemInfo computeSplitWorkItemInfo(const SwitchWorkListItem &W);
   virtual ~SwitchLowering() = default;
 
 private:
diff --git a/llvm/include/llvm/DWARFLinkerParallel/AddressesMap.h b/llvm/include/llvm/DWARFLinker/AddressesMap.h
index b451fee4e0b7..d8b3b4407471 100644
--- a/llvm/include/llvm/DWARFLinkerParallel/AddressesMap.h
+++ b/llvm/include/llvm/DWARFLinker/AddressesMap.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKERPARALLEL_ADDRESSESMAP_H
-#define LLVM_DWARFLINKERPARALLEL_ADDRESSESMAP_H
+#ifndef LLVM_DWARFLINKER_ADDRESSESMAP_H
+#define LLVM_DWARFLINKER_ADDRESSESMAP_H
 
 #include "llvm/ADT/AddressRanges.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
@@ -17,7 +17,7 @@
 #include <cstdint>
 
 namespace llvm {
-namespace dwarflinker_parallel {
+namespace dwarf_linker {
 
 /// Mapped value in the address map is the offset to apply to the
 /// linked address.
@@ -186,7 +186,7 @@ protected:
   }
 };
 
-} // end of namespace dwarflinker_parallel
+} // namespace dwarf_linker
 } // end namespace llvm
 
-#endif // LLVM_DWARFLINKERPARALLEL_ADDRESSESMAP_H
+#endif // LLVM_DWARFLINKER_ADDRESSESMAP_H
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h
index 2bd85e30d3b1..d3aaa3baadc4 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinker.h
+++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h
@@ -6,14 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKER_DWARFLINKER_H
-#define LLVM_DWARFLINKER_DWARFLINKER_H
+#ifndef LLVM_DWARFLINKER_CLASSIC_DWARFLINKER_H
+#define LLVM_DWARFLINKER_CLASSIC_DWARFLINKER_H
 
 #include "llvm/ADT/AddressRanges.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/AccelTable.h"
 #include "llvm/CodeGen/NonRelocatableStringpool.h"
-#include "llvm/DWARFLinker/DWARFLinkerCompileUnit.h"
+#include "llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h"
+#include "llvm/DWARFLinker/DWARFLinkerBase.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
@@ -25,73 +26,11 @@ namespace llvm {
 class DWARFExpression;
 class DWARFUnit;
 class DataExtractor;
-class DeclContextTree;
 template <typename T> class SmallVectorImpl;
 
-enum class DwarfLinkerClient { Dsymutil, LLD, General };
-
-/// AddressesMap represents information about valid addresses used
-/// by debug information. Valid addresses are those which points to
-/// live code sections. i.e. relocations for these addresses point
-/// into sections which would be/are placed into resulting binary.
-class AddressesMap {
-public:
-  virtual ~AddressesMap();
-
-  /// Checks that there are valid relocations against a .debug_info
-  /// section.
-  virtual bool hasValidRelocs() = 0;
-
-  /// Checks that the specified DWARF expression operand \p Op references live
-  /// code section and returns the relocation adjustment value (to get the
-  /// linked address this value might be added to the source expression operand
-  /// address).
-  /// \returns relocation adjustment value or std::nullopt if there is no
-  /// corresponding live address.
-  virtual std::optional<int64_t>
-  getExprOpAddressRelocAdjustment(DWARFUnit &U,
-                                  const DWARFExpression::Operation &Op,
-                                  uint64_t StartOffset, uint64_t EndOffset) = 0;
-
-  /// Checks that the specified subprogram \p DIE references the live code
-  /// section and returns the relocation adjustment value (to get the linked
-  /// address this value might be added to the source subprogram address).
-  /// Allowed kinds of input DIE: DW_TAG_subprogram, DW_TAG_label.
-  /// \returns relocation adjustment value or std::nullopt if there is no
-  /// corresponding live address.
-  virtual std::optional<int64_t>
-  getSubprogramRelocAdjustment(const DWARFDie &DIE) = 0;
-
-  /// Returns the file name associated to the AddessesMap
-  virtual std::optional<StringRef> getLibraryInstallName() = 0;
-
-  /// Apply the valid relocations to the buffer \p Data, taking into
-  /// account that Data is at \p BaseOffset in the .debug_info section.
-  ///
-  /// \returns true whether any reloc has been applied.
-  virtual bool applyValidRelocs(MutableArrayRef<char> Data, uint64_t BaseOffset,
-                                bool IsLittleEndian) = 0;
-
-  /// Check if the linker needs to gather and save relocation info.
-  virtual bool needToSaveValidRelocs() = 0;
-
-  /// Update and save original relocations located in between StartOffset and
-  /// EndOffset. LinkedOffset is the value which should be added to the original
-  /// relocation offset to get new relocation offset in linked binary.
-  virtual void updateAndSaveValidRelocs(bool IsDWARF5,
-                                        uint64_t OriginalUnitOffset,
-                                        int64_t LinkedOffset,
-                                        uint64_t StartOffset,
-                                        uint64_t EndOffset) = 0;
-
-  /// Update the valid relocations that used OriginalUnitOffset as the compile
-  /// unit offset, and update their values to reflect OutputUnitOffset.
-  virtual void updateRelocationsWithUnitOffset(uint64_t OriginalUnitOffset,
-                                               uint64_t OutputUnitOffset) = 0;
-
-  /// Erases all data.
-  virtual void clear() = 0;
-};
+namespace dwarf_linker {
+namespace classic {
+class DeclContextTree;
 
 using Offset2UnitMap = DenseMap<uint64_t, CompileUnit *>;
 
@@ -117,7 +56,7 @@ struct DebugDieValuePool {
 /// DwarfEmitter presents interface to generate all debug info tables.
 class DwarfEmitter {
 public:
-  virtual ~DwarfEmitter();
+  virtual ~DwarfEmitter() = default;
 
   /// Emit section named SecName with data SecData.
   virtual void emitSectionContents(StringRef SecData, StringRef SecName) = 0;
@@ -282,44 +221,6 @@ public:
 class DwarfStreamer;
 using UnitListTy = std::vector<std::unique_ptr<CompileUnit>>;
 
-/// This class represents DWARF information for source file
-/// and its address map.
-class DWARFFile {
-public:
-  using UnloadCallbackTy = std::function<void(StringRef FileName)>;
-  DWARFFile(StringRef Name, std::unique_ptr<DWARFContext> Dwarf,
-            std::unique_ptr<AddressesMap> Addresses,
-            UnloadCallbackTy UnloadFunc = nullptr)
-      : FileName(Name), Dwarf(std::move(Dwarf)),
-        Addresses(std::move(Addresses)), UnloadFunc(UnloadFunc) {}
-
-  /// The object file name.
-  StringRef FileName;
-
-  /// The source DWARF information.
-  std::unique_ptr<DWARFContext> Dwarf;
-
-  /// Helpful address information(list of valid address ranges, relocations).
-  std::unique_ptr<AddressesMap> Addresses;
-
-  /// Callback to the module keeping object file to unload.
-  UnloadCallbackTy UnloadFunc;
-
-  /// Unloads object file and corresponding AddressesMap and Dwarf Context.
-  void unload() {
-    Addresses.reset();
-    Dwarf.reset();
-
-    if (UnloadFunc)
-      UnloadFunc(FileName);
-  }
-};
-
-typedef std::map<std::string, std::string> swiftInterfacesMap;
-typedef std::map<std::string, std::string> objectPrefixMap;
-
-typedef function_ref<void(const DWARFUnit &Unit)> CompileUnitHandler;
-
 /// The core of the Dwarf linking logic.
 ///
 /// The generation of the dwarf information from the object files will be
@@ -334,41 +235,20 @@ typedef function_ref<void(const DWARFUnit &Unit)> CompileUnitHandler;
 /// a variable). These relocations are called ValidRelocs in the
 /// AddressesInfo and are gathered as a very first step when we start
 /// processing a object file.
-class DWARFLinker {
+class DWARFLinker : public DWARFLinkerBase {
 public:
-  typedef std::function<void(const Twine &Warning, StringRef Context,
-                             const DWARFDie *DIE)>
-      messageHandler;
-  DWARFLinker(messageHandler ErrorHandler, messageHandler WarningHandler,
+  DWARFLinker(MessageHandlerTy ErrorHandler, MessageHandlerTy WarningHandler,
               std::function<StringRef(StringRef)> StringsTranslator)
-      : DwarfLinkerClientID(DwarfLinkerClient::Dsymutil),
-        StringsTranslator(StringsTranslator), ErrorHandler(ErrorHandler),
+      : StringsTranslator(StringsTranslator), ErrorHandler(ErrorHandler),
         WarningHandler(WarningHandler) {}
 
   static std::unique_ptr<DWARFLinker> createLinker(
-      messageHandler ErrorHandler, messageHandler WarningHandler,
+      MessageHandlerTy ErrorHandler, MessageHandlerTy WarningHandler,
       std::function<StringRef(StringRef)> StringsTranslator = nullptr) {
     return std::make_unique<DWARFLinker>(ErrorHandler, WarningHandler,
                                          StringsTranslator);
   }
 
-  /// Type of output file.
-  enum class OutputFileType {
-    Object,
-    Assembly,
-  };
-
-  /// The kind of accelerator tables we should emit.
-  enum class AccelTableKind : uint8_t {
-    Apple,     ///< .apple_names, .apple_namespaces, .apple_types, .apple_objc.
-    Pub,       ///< .debug_pubnames, .debug_pubtypes
-    DebugNames ///< .debug_names.
-  };
-  typedef std::function<void(const DWARFFile &File, llvm::StringRef Output)> inputVerificationHandler;
-  typedef std::function<ErrorOr<DWARFFile &>(StringRef ContainerName,
-                                             StringRef Path)>
-      objFileLoader;
-
   Error createEmitter(const Triple &TheTriple, OutputFileType FileType,
                       raw_pwrite_stream &OutFile);
 
@@ -381,73 +261,82 @@ public:
   ///
   /// \pre NoODR, Update options should be set before call to addObjectFile.
   void addObjectFile(
-      DWARFFile &File, objFileLoader Loader = nullptr,
-      CompileUnitHandler OnCUDieLoaded = [](const DWARFUnit &) {});
+      DWARFFile &File, ObjFileLoaderTy Loader = nullptr,
+      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {}) override;
 
   /// Link debug info for added objFiles. Object files are linked all together.
-  Error link();
+  Error link() override;
 
   /// A number of methods setting various linking options:
 
   /// Allows to generate log of linking process to the standard output.
-  void setVerbosity(bool Verbose) { Options.Verbose = Verbose; }
+  void setVerbosity(bool Verbose) override { Options.Verbose = Verbose; }
 
   /// Print statistics to standard output.
-  void setStatistics(bool Statistics) { Options.Statistics = Statistics; }
+  void setStatistics(bool Statistics) override {
+    Options.Statistics = Statistics;
+  }
 
   /// Verify the input DWARF.
-  void setVerifyInputDWARF(bool Verify) { Options.VerifyInputDWARF = Verify; }
+  void setVerifyInputDWARF(bool Verify) override {
+    Options.VerifyInputDWARF = Verify;
+  }
 
   /// Do not unique types according to ODR.
-  void setNoODR(bool NoODR) { Options.NoODR = NoODR; }
+  void setNoODR(bool NoODR) override { Options.NoODR = NoODR; }
 
   /// Update index tables only(do not modify rest of DWARF).
-  void setUpdateIndexTablesOnly(bool Update) { Options.Update = Update; }
+  void setUpdateIndexTablesOnly(bool Update) override {
+    Options.Update = Update;
+  }
 
   /// Allow generating valid, but non-deterministic output.
-  void setAllowNonDeterministicOutput(bool) { /* Nothing to do. */
+  void setAllowNonDeterministicOutput(bool) override { /* Nothing to do. */
   }
 
   /// Set whether to keep the enclosing function for a static variable.
-  void setKeepFunctionForStatic(bool KeepFunctionForStatic) {
+  void setKeepFunctionForStatic(bool KeepFunctionForStatic) override {
     Options.KeepFunctionForStatic = KeepFunctionForStatic;
   }
 
   /// Use specified number of threads for parallel files linking.
-  void setNumThreads(unsigned NumThreads) { Options.Threads = NumThreads; }
+  void setNumThreads(unsigned NumThreads) override {
+    Options.Threads = NumThreads;
+  }
 
   /// Add kind of accelerator tables to be generated.
-  void addAccelTableKind(AccelTableKind Kind) {
+  void addAccelTableKind(AccelTableKind Kind) override {
     assert(!llvm::is_contained(Options.AccelTables, Kind));
     Options.AccelTables.emplace_back(Kind);
   }
 
   /// Set prepend path for clang modules.
-  void setPrependPath(const std::string &Ppath) { Options.PrependPath = Ppath; }
+  void setPrependPath(StringRef Ppath) override { Options.PrependPath = Ppath; }
 
   /// Set estimated objects files amount, for preliminary data allocation.
-  void setEstimatedObjfilesAmount(unsigned ObjFilesNum) {
+  void setEstimatedObjfilesAmount(unsigned ObjFilesNum) override {
     ObjectContexts.reserve(ObjFilesNum);
   }
 
   /// Set verification handler which would be used to report verification
   /// errors.
-  void setInputVerificationHandler(inputVerificationHandler Handler) {
+  void
+  setInputVerificationHandler(InputVerificationHandlerTy Handler) override {
     Options.InputVerificationHandler = Handler;
   }
 
   /// Set map for Swift interfaces.
-  void setSwiftInterfacesMap(swiftInterfacesMap *Map) {
+  void setSwiftInterfacesMap(SwiftInterfacesMapTy *Map) override {
     Options.ParseableSwiftInterfaces = Map;
   }
 
   /// Set prefix map for objects.
-  void setObjectPrefixMap(objectPrefixMap *Map) {
+  void setObjectPrefixMap(ObjectPrefixMapTy *Map) override {
     Options.ObjectPrefixMap = Map;
   }
 
   /// Set target DWARF version.
-  Error setTargetDWARFVersion(uint16_t TargetDWARFVersion) {
+  Error setTargetDWARFVersion(uint16_t TargetDWARFVersion) override {
     if ((TargetDWARFVersion < 1) || (TargetDWARFVersion > 5))
       return createStringError(std::errc::invalid_argument,
                                "unsupported DWARF version: %d",
@@ -619,16 +508,17 @@ private:
   /// pointing to the module, and a DW_AT_gnu_dwo_id with the module
   /// hash.
   bool registerModuleReference(const DWARFDie &CUDie, LinkContext &Context,
-                               objFileLoader Loader,
-                               CompileUnitHandler OnCUDieLoaded,
+                               ObjFileLoaderTy Loader,
+                               CompileUnitHandlerTy OnCUDieLoaded,
                                unsigned Indent = 0);
 
   /// Recursively add the debug info in this clang module .pcm
   /// file (and all the modules imported by it in a bottom-up fashion)
   /// to ModuleUnits.
-  Error loadClangModule(objFileLoader Loader, const DWARFDie &CUDie,
+  Error loadClangModule(ObjFileLoaderTy Loader, const DWARFDie &CUDie,
                         const std::string &PCMFile, LinkContext &Context,
-                        CompileUnitHandler OnCUDieLoaded, unsigned Indent = 0);
+                        CompileUnitHandlerTy OnCUDieLoaded,
+                        unsigned Indent = 0);
 
   /// Clone specified Clang module unit \p Unit.
   Error cloneModuleUnit(LinkContext &Context, RefModuleUnit &Unit,
@@ -911,18 +801,16 @@ private:
   /// Mapping the PCM filename to the DwoId.
   StringMap<uint64_t> ClangModules;
 
-  DwarfLinkerClient DwarfLinkerClientID;
-
   std::function<StringRef(StringRef)> StringsTranslator = nullptr;
 
   /// A unique ID that identifies each compile unit.
   unsigned UniqueUnitID = 0;
 
   // error handler
-  messageHandler ErrorHandler = nullptr;
+  MessageHandlerTy ErrorHandler = nullptr;
 
   // warning handler
-  messageHandler WarningHandler = nullptr;
+  MessageHandlerTy WarningHandler = nullptr;
 
   /// linking options
   struct DWARFLinkerOptions {
@@ -958,20 +846,22 @@ private:
     std::string PrependPath;
 
     // input verification handler
-    inputVerificationHandler InputVerificationHandler = nullptr;
+    InputVerificationHandlerTy InputVerificationHandler = nullptr;
 
     /// A list of all .swiftinterface files referenced by the debug
     /// info, mapping Module name to path on disk. The entries need to
     /// be uniqued and sorted and there are only few entries expected
     /// per compile unit, which is why this is a std::map.
     /// this is dsymutil specific fag.
-    swiftInterfacesMap *ParseableSwiftInterfaces = nullptr;
+    SwiftInterfacesMapTy *ParseableSwiftInterfaces = nullptr;
 
     /// A list of remappings to apply to file paths.
-    objectPrefixMap *ObjectPrefixMap = nullptr;
+    ObjectPrefixMapTy *ObjectPrefixMap = nullptr;
   } Options;
 };
 
-} // end namespace llvm
+} // end of namespace classic
+} // end of namespace dwarf_linker
+} // end of namespace llvm
 
-#endif // LLVM_DWARFLINKER_DWARFLINKER_H
+#endif // LLVM_DWARFLINKER_CLASSIC_DWARFLINKER_H
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h
index 08ebd4bc70bc..bfe544946fd9 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h
+++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H
-#define LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H
+#ifndef LLVM_DWARFLINKER_CLASSIC_DWARFLINKERCOMPILEUNIT_H
+#define LLVM_DWARFLINKER_CLASSIC_DWARFLINKERCOMPILEUNIT_H
 
 #include "llvm/ADT/AddressRanges.h"
 #include "llvm/ADT/DenseMap.h"
@@ -16,6 +16,8 @@
 #include <optional>
 
 namespace llvm {
+namespace dwarf_linker {
+namespace classic {
 
 class DeclContext;
 
@@ -327,6 +329,8 @@ private:
   std::string ClangModuleName;
 };
 
-} // end namespace llvm
+} // end of namespace classic
+} // end of namespace dwarf_linker
+} // end of namespace llvm
 
-#endif // LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H
+#endif // LLVM_DWARFLINKER_CLASSIC_DWARFLINKERCOMPILEUNIT_H
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerDeclContext.h
index fb02b0fc1b4d..b00f68c3be84 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h
+++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerDeclContext.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKER_DWARFLINKERDECLCONTEXT_H
-#define LLVM_DWARFLINKER_DWARFLINKERDECLCONTEXT_H
+#ifndef LLVM_DWARFLINKER_CLASSIC_DWARFLINKERDECLCONTEXT_H
+#define LLVM_DWARFLINKER_CLASSIC_DWARFLINKERDECLCONTEXT_H
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseMapInfo.h"
@@ -21,6 +21,8 @@
 #include <atomic>
 
 namespace llvm {
+namespace dwarf_linker {
+namespace classic {
 
 class CompileUnit;
 struct DeclMapInfo;
@@ -184,6 +186,8 @@ struct DeclMapInfo : private DenseMapInfo<DeclContext *> {
   }
 };
 
-} // end namespace llvm
+} // end of namespace classic
+} // end of namespace dwarf_linker
+} // end of namespace llvm
 
-#endif // LLVM_DWARFLINKER_DWARFLINKERDECLCONTEXT_H
+#endif // LLVM_DWARFLINKER_CLASSIC_DWARFLINKERDECLCONTEXT_H
diff --git a/llvm/include/llvm/DWARFLinker/DWARFStreamer.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFStreamer.h
index 18eb7277bfa2..f010c348f121 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFStreamer.h
+++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFStreamer.h
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKER_DWARFSTREAMER_H
-#define LLVM_DWARFLINKER_DWARFSTREAMER_H
+#ifndef LLVM_DWARFLINKER_CLASSIC_DWARFSTREAMER_H
+#define LLVM_DWARFLINKER_CLASSIC_DWARFSTREAMER_H
 
+#include "DWARFLinker.h"
 #include "llvm/BinaryFormat/Swift.h"
 #include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/DWARFLinker/DWARFLinker.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -23,6 +23,12 @@
 namespace llvm {
 template <typename DataT> class AccelTable;
 
+class MCCodeEmitter;
+class DWARFDebugMacro;
+
+namespace dwarf_linker {
+namespace classic {
+
 ///   User of DwarfStreamer should call initialization code
 ///   for AsmPrinter:
 ///
@@ -31,21 +37,19 @@ template <typename DataT> class AccelTable;
 ///   InitializeAllTargets();
 ///   InitializeAllAsmPrinters();
 
-class MCCodeEmitter;
-class DWARFDebugMacro;
-
 /// The Dwarf streaming logic.
 ///
 /// All interactions with the MC layer that is used to build the debug
 /// information binary representation are handled in this class.
 class DwarfStreamer : public DwarfEmitter {
 public:
-  DwarfStreamer(DWARFLinker::OutputFileType OutFileType,
+  DwarfStreamer(DWARFLinkerBase::OutputFileType OutFileType,
                 raw_pwrite_stream &OutFile,
                 std::function<StringRef(StringRef Input)> Translator,
-                DWARFLinker::messageHandler Warning)
+                DWARFLinkerBase::MessageHandlerTy Warning)
       : OutFile(OutFile), OutFileType(OutFileType), Translator(Translator),
         WarningHandler(Warning) {}
+  virtual ~DwarfStreamer() = default;
 
   Error init(Triple TheTriple, StringRef Swift5ReflectionSegmentName);
 
@@ -310,9 +314,11 @@ private:
                              const CompileUnit &Unit,
                              const std::vector<CompileUnit::AccelInfo> &Names);
 
-  DWARFLinker::messageHandler WarningHandler = nullptr;
+  DWARFLinkerBase::MessageHandlerTy WarningHandler = nullptr;
 };
 
-} // end namespace llvm
+} // end of namespace classic
+} // end of namespace dwarf_linker
+} // end of namespace llvm
 
-#endif // LLVM_DWARFLINKER_DWARFSTREAMER_H
+#endif // LLVM_DWARFLINKER_CLASSIC_DWARFSTREAMER_H
diff --git a/llvm/include/llvm/DWARFLinkerParallel/DWARFFile.h b/llvm/include/llvm/DWARFLinker/DWARFFile.h
index c320530569bb..c1d0fd87c7d7 100644
--- a/llvm/include/llvm/DWARFLinkerParallel/DWARFFile.h
+++ b/llvm/include/llvm/DWARFLinker/DWARFFile.h
@@ -6,18 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKERPARALLEL_DWARFFILE_H
-#define LLVM_DWARFLINKERPARALLEL_DWARFFILE_H
+#ifndef LLVM_DWARFLINKER_DWARFFILE_H
+#define LLVM_DWARFLINKER_DWARFFILE_H
 
+#include "AddressesMap.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/DWARFLinkerParallel/AddressesMap.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
-#include "llvm/Support/Endian.h"
 #include <functional>
 #include <memory>
 
 namespace llvm {
-namespace dwarflinker_parallel {
+namespace dwarf_linker {
 
 /// This class represents DWARF information for source file
 /// and it's address map.
@@ -29,7 +28,9 @@ public:
 
   DWARFFile(StringRef Name, std::unique_ptr<DWARFContext> Dwarf,
             std::unique_ptr<AddressesMap> Addresses,
-            UnloadCallbackTy UnloadFunc = nullptr);
+            UnloadCallbackTy UnloadFunc = nullptr)
+      : FileName(Name), Dwarf(std::move(Dwarf)),
+        Addresses(std::move(Addresses)), UnloadFunc(UnloadFunc) {}
 
   /// Object file name.
   StringRef FileName;
@@ -53,7 +54,7 @@ public:
   }
 };
 
-} // end namespace dwarflinker_parallel
+} // namespace dwarf_linker
 } // end namespace llvm
 
-#endif // LLVM_DWARFLINKERPARALLEL_DWARFFILE_H
+#endif // LLVM_DWARFLINKER_DWARFFILE_H
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h
new file mode 100644
index 000000000000..626fb53d90f9
--- /dev/null
+++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h
@@ -0,0 +1,100 @@
+//===- DWARFLinkerBase.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DWARFLINKER_DWARFLINKERBASE_H
+#define LLVM_DWARFLINKER_DWARFLINKERBASE_H
+#include "AddressesMap.h"
+#include "DWARFFile.h"
+#include "llvm/ADT/AddressRanges.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
+#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
+#include "llvm/DebugInfo/DWARF/DWARFDie.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include <map>
+namespace llvm {
+class DWARFUnit;
+
+namespace dwarf_linker {
+
+/// The base interface for DWARFLinker implementations.
+class DWARFLinkerBase {
+public:
+  virtual ~DWARFLinkerBase() = default;
+  using MessageHandlerTy = std::function<void(
+      const Twine &Warning, StringRef Context, const DWARFDie *DIE)>;
+  using ObjFileLoaderTy = std::function<ErrorOr<DWARFFile &>(
+      StringRef ContainerName, StringRef Path)>;
+  using InputVerificationHandlerTy =
+      std::function<void(const DWARFFile &File, llvm::StringRef Output)>;
+  using ObjectPrefixMapTy = std::map<std::string, std::string>;
+  using CompileUnitHandlerTy = function_ref<void(const DWARFUnit &Unit)>;
+  using TranslatorFuncTy = std::function<StringRef(StringRef)>;
+  using SwiftInterfacesMapTy = std::map<std::string, std::string>;
+  /// Type of output file.
+  enum class OutputFileType : uint8_t {
+    Object,
+    Assembly,
+  };
+  /// The kind of accelerator tables to be emitted.
+  enum class AccelTableKind : uint8_t {
+    Apple,     ///< .apple_names, .apple_namespaces, .apple_types, .apple_objc.
+    Pub,       ///< .debug_pubnames, .debug_pubtypes
+    DebugNames ///< .debug_names.
+  };
+  /// Add an object file to be linked. Pre-load compile unit die. Call
+  /// \p OnCUDieLoaded for each compile unit die. If \p File has reference to
+  /// a Clang module and UpdateIndexTablesOnly == false then the module is be
+  /// pre-loaded by \p Loader.
+  ///
+  /// \pre a call to setNoODR(true) and/or setUpdateIndexTablesOnly(bool Update)
+  ///      must be made when required.
+  virtual void addObjectFile(
+      DWARFFile &File, ObjFileLoaderTy Loader = nullptr,
+      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {}) = 0;
+  /// Link the debug info for all object files added through calls to
+  /// addObjectFile.
+  virtual Error link() = 0;
+  /// A number of methods setting various linking options:
+  /// Enable logging to standard output.
+  virtual void setVerbosity(bool Verbose) = 0;
+  /// Print statistics to standard output.
+  virtual void setStatistics(bool Statistics) = 0;
+  /// Verify the input DWARF.
+  virtual void setVerifyInputDWARF(bool Verify) = 0;
+  /// Do not unique types according to ODR.
+  virtual void setNoODR(bool NoODR) = 0;
+  /// Update index tables only (do not modify rest of DWARF).
+  virtual void setUpdateIndexTablesOnly(bool Update) = 0;
+  /// Allows generating non-deterministic output in exchange for more
+  /// parallelism.
+  virtual void setAllowNonDeterministicOutput(bool) = 0;
+  /// Set whether to keep the enclosing function for a static variable.
+  virtual void setKeepFunctionForStatic(bool KeepFunctionForStatic) = 0;
+  /// Use specified number of threads for parallel files linking.
+  virtual void setNumThreads(unsigned NumThreads) = 0;
+  /// Add kind of accelerator tables to be generated.
+  virtual void addAccelTableKind(AccelTableKind Kind) = 0;
+  /// Set prepend path for clang modules.
+  virtual void setPrependPath(StringRef Ppath) = 0;
+  /// Set estimated objects files amount, for preliminary data allocation.
+  virtual void setEstimatedObjfilesAmount(unsigned ObjFilesNum) = 0;
+  /// Set verification handler used to report verification errors.
+  virtual void
+  setInputVerificationHandler(InputVerificationHandlerTy Handler) = 0;
+  /// Set map for Swift interfaces.
+  virtual void setSwiftInterfacesMap(SwiftInterfacesMapTy *Map) = 0;
+  /// Set prefix map for objects.
+  virtual void setObjectPrefixMap(ObjectPrefixMapTy *Map) = 0;
+  /// Set target DWARF version.
+  virtual Error setTargetDWARFVersion(uint16_t TargetDWARFVersion) = 0;
+};
+} // end namespace dwarf_linker
+} // end namespace llvm
+#endif // LLVM_DWARFLINKER_DWARFLINKERBASE_H
diff --git a/llvm/include/llvm/DWARFLinkerParallel/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/Parallel/DWARFLinker.h
index c16c94d65c2f..c38a9906940e 100644
--- a/llvm/include/llvm/DWARFLinkerParallel/DWARFLinker.h
+++ b/llvm/include/llvm/DWARFLinker/Parallel/DWARFLinker.h
@@ -6,11 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKERPARALLEL_DWARFLINKER_H
-#define LLVM_DWARFLINKERPARALLEL_DWARFLINKER_H
+#ifndef LLVM_DWARFLINKER_PARALLEL_DWARFLINKER_H
+#define LLVM_DWARFLINKER_PARALLEL_DWARFLINKER_H
 
 #include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/DWARFLinkerParallel/DWARFFile.h"
+#include "llvm/DWARFLinker/DWARFFile.h"
+#include "llvm/DWARFLinker/DWARFLinkerBase.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
 #include "llvm/MC/MCDwarf.h"
@@ -85,7 +86,8 @@
 ///
 
 namespace llvm {
-namespace dwarflinker_parallel {
+namespace dwarf_linker {
+namespace parallel {
 
 /// ExtraDwarfEmitter allows adding extra data to the DWARFLinker output.
 /// The finish() method should be called after all extra data are emitted.
@@ -111,31 +113,8 @@ public:
   virtual AsmPrinter &getAsmPrinter() const = 0;
 };
 
-class DWARFLinker {
+class DWARFLinker : public DWARFLinkerBase {
 public:
-  /// Type of output file.
-  enum class OutputFileType {
-    Object,
-    Assembly,
-  };
-
-  /// The kind of accelerator tables we should emit.
-  enum class AccelTableKind : uint8_t {
-    Apple,     ///< .apple_names, .apple_namespaces, .apple_types, .apple_objc.
-    Pub,       ///< .debug_pubnames, .debug_pubtypes
-    DebugNames ///< .debug_names.
-  };
-
-  using MessageHandlerTy = std::function<void(
-      const Twine &Warning, StringRef Context, const DWARFDie *DIE)>;
-  using ObjFileLoaderTy = std::function<ErrorOr<DWARFFile &>(
-      StringRef ContainerName, StringRef Path)>;
-  using InputVerificationHandlerTy = std::function<void(const DWARFFile &File, llvm::StringRef Output)>;
-  using ObjectPrefixMapTy = std::map<std::string, std::string>;
-  using CompileUnitHandlerTy = function_ref<void(const DWARFUnit &Unit)>;
-  using TranslatorFuncTy = std::function<StringRef(StringRef)>;
-  using SwiftInterfacesMapTy = std::map<std::string, std::string>;
-
   virtual ~DWARFLinker() = default;
 
   /// Creates dwarf linker instance.
@@ -149,75 +128,10 @@ public:
 
   /// Returns previously created dwarf emitter. May be nullptr.
   virtual ExtraDwarfEmitter *getEmitter() = 0;
-
-  /// Add object file to be linked. Pre-load compile unit die. Call
-  /// \p OnCUDieLoaded for each compile unit die. If specified \p File
-  /// has reference to the Clang module then such module would be
-  /// pre-loaded by \p Loader for !Update case.
-  ///
-  /// \pre NoODR, Update options should be set before call to addObjectFile.
-  virtual void addObjectFile(
-      DWARFFile &File, ObjFileLoaderTy Loader = nullptr,
-      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {}) = 0;
-
-  /// Link debug info for added files.
-  virtual Error link() = 0;
-
-  /// \defgroup Methods setting various linking options:
-  ///
-  /// @{
-
-  /// Allows to generate log of linking process to the standard output.
-  virtual void setVerbosity(bool Verbose) = 0;
-
-  /// Print statistics to standard output.
-  virtual void setStatistics(bool Statistics) = 0;
-
-  /// Verify the input DWARF.
-  virtual void setVerifyInputDWARF(bool Verify) = 0;
-
-  /// Do not unique types according to ODR.
-  virtual void setNoODR(bool NoODR) = 0;
-
-  /// Update index tables only(do not modify rest of DWARF).
-  virtual void setUpdateIndexTablesOnly(bool UpdateIndexTablesOnly) = 0;
-
-  /// Allow generating valid, but non-deterministic output.
-  virtual void
-  setAllowNonDeterministicOutput(bool AllowNonDeterministicOutput) = 0;
-
-  /// Set to keep the enclosing function for a static variable.
-  virtual void setKeepFunctionForStatic(bool KeepFunctionForStatic) = 0;
-
-  /// Use specified number of threads for parallel files linking.
-  virtual void setNumThreads(unsigned NumThreads) = 0;
-
-  /// Add kind of accelerator tables to be generated.
-  virtual void addAccelTableKind(AccelTableKind Kind) = 0;
-
-  /// Set prepend path for clang modules.
-  virtual void setPrependPath(const std::string &Ppath) = 0;
-
-  /// Set estimated objects files amount, for preliminary data allocation.
-  virtual void setEstimatedObjfilesAmount(unsigned ObjFilesNum) = 0;
-
-  /// Set verification handler which would be used to report verification
-  /// errors.
-  virtual void
-  setInputVerificationHandler(InputVerificationHandlerTy Handler) = 0;
-
-  /// Set map for Swift interfaces.
-  virtual void setSwiftInterfacesMap(SwiftInterfacesMapTy *Map) = 0;
-
-  /// Set prefix map for objects.
-  virtual void setObjectPrefixMap(ObjectPrefixMapTy *Map) = 0;
-
-  /// Set target DWARF version.
-  virtual Error setTargetDWARFVersion(uint16_t TargetDWARFVersion) = 0;
-  /// @}
 };
 
-} // end namespace dwarflinker_parallel
-} // end namespace llvm
+} // end of namespace parallel
+} // end of namespace dwarf_linker
+} // end of namespace llvm
 
-#endif // LLVM_DWARFLINKERPARALLEL_DWARFLINKER_H
+#endif // LLVM_DWARFLINKER_PARALLEL_DWARFLINKER_H
diff --git a/llvm/include/llvm/DWARFLinkerParallel/StringPool.h b/llvm/include/llvm/DWARFLinker/StringPool.h
index e55909f34311..d0f4e211fac3 100644
--- a/llvm/include/llvm/DWARFLinkerParallel/StringPool.h
+++ b/llvm/include/llvm/DWARFLinker/StringPool.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DWARFLINKERPARALLEL_STRINGPOOL_H
-#define LLVM_DWARFLINKERPARALLEL_STRINGPOOL_H
+#ifndef LLVM_DWARFLINKER_STRINGPOOL_H
+#define LLVM_DWARFLINKER_STRINGPOOL_H
 
 #include "llvm/ADT/ConcurrentHashtable.h"
 #include "llvm/CodeGen/DwarfStringPoolEntry.h"
@@ -16,7 +16,7 @@
 #include <string_view>
 
 namespace llvm {
-namespace dwarflinker_parallel {
+namespace dwarf_linker {
 
 /// StringEntry keeps data of the string: the length, external offset
 /// and a string body which is placed right after StringEntry.
@@ -41,35 +41,38 @@ public:
 
   /// \returns newly created object of KeyDataTy type.
   static inline StringEntry *
-  create(const StringRef &Key, parallel::PerThreadBumpPtrAllocator &Allocator) {
+  create(const StringRef &Key,
+         llvm::parallel::PerThreadBumpPtrAllocator &Allocator) {
     return StringEntry::create(Key, Allocator);
   }
 };
 
 class StringPool
     : public ConcurrentHashTableByPtr<StringRef, StringEntry,
-                                      parallel::PerThreadBumpPtrAllocator,
+                                      llvm::parallel::PerThreadBumpPtrAllocator,
                                       StringPoolEntryInfo> {
 public:
   StringPool()
       : ConcurrentHashTableByPtr<StringRef, StringEntry,
-                                 parallel::PerThreadBumpPtrAllocator,
+                                 llvm::parallel::PerThreadBumpPtrAllocator,
                                  StringPoolEntryInfo>(Allocator) {}
 
   StringPool(size_t InitialSize)
       : ConcurrentHashTableByPtr<StringRef, StringEntry,
-                                 parallel::PerThreadBumpPtrAllocator,
+                                 llvm::parallel::PerThreadBumpPtrAllocator,
                                  StringPoolEntryInfo>(Allocator, InitialSize) {}
 
-  parallel::PerThreadBumpPtrAllocator &getAllocatorRef() { return Allocator; }
+  llvm::parallel::PerThreadBumpPtrAllocator &getAllocatorRef() {
+    return Allocator;
+  }
 
   void clear() { Allocator.Reset(); }
 
 private:
-  parallel::PerThreadBumpPtrAllocator Allocator;
+  llvm::parallel::PerThreadBumpPtrAllocator Allocator;
 };
 
-} // end of namespace dwarflinker_parallel
+} // namespace dwarf_linker
 } // end namespace llvm
 
-#endif // LLVM_DWARFLINKERPARALLEL_STRINGPOOL_H
+#endif // LLVM_DWARFLINKER_STRINGPOOL_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
index 6b1b2ae6d7e0..df862f60cb2f 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
@@ -18,11 +18,16 @@
 namespace llvm {
 
 class raw_ostream;
+
+namespace dwarf_linker {
+namespace classic {
 class DwarfStreamer;
+}
+} // namespace dwarf_linker
 
 class DWARFDebugMacro {
-  friend DwarfStreamer;
-  friend dwarflinker_parallel::CompileUnit;
+  friend dwarf_linker::classic::DwarfStreamer;
+  friend dwarf_linker::parallel::CompileUnit;
 
   /// DWARFv5 section 6.3.1 Macro Information Header.
   enum HeaderFlagMask {
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 7084081ce61a..f20e71781f46 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -43,9 +43,11 @@ class DWARFObject;
 class raw_ostream;
 struct DIDumpOptions;
 struct DWARFSection;
-namespace dwarflinker_parallel {
+namespace dwarf_linker {
+namespace parallel {
 class CompileUnit;
 }
+} // namespace dwarf_linker
 
 /// Base class describing the header of any kind of "unit."  Some information
 /// is specific to certain unit types.  We separate this class out so we can
@@ -256,7 +258,7 @@ class DWARFUnit {
   std::shared_ptr<DWARFUnit> DWO;
 
 protected:
-  friend dwarflinker_parallel::CompileUnit;
+  friend dwarf_linker::parallel::CompileUnit;
 
   /// Return the index of a \p Die entry inside the unit's DIE vector.
   ///
diff --git a/llvm/include/llvm/Demangle/Demangle.h b/llvm/include/llvm/Demangle/Demangle.h
index 70cfc1418f0c..fe129603c078 100644
--- a/llvm/include/llvm/Demangle/Demangle.h
+++ b/llvm/include/llvm/Demangle/Demangle.h
@@ -32,7 +32,7 @@ enum : int {
 /// Returns a non-NULL pointer to a NUL-terminated C style string
 /// that should be explicitly freed, if successful. Otherwise, may return
 /// nullptr if mangled_name is not a valid mangling or is nullptr.
-char *itaniumDemangle(std::string_view mangled_name);
+char *itaniumDemangle(std::string_view mangled_name, bool ParseParams = true);
 
 enum MSDemangleFlags {
   MSDF_None = 0,
@@ -68,7 +68,8 @@ char *dlangDemangle(std::string_view MangledName);
 std::string demangle(std::string_view MangledName);
 
 bool nonMicrosoftDemangle(std::string_view MangledName, std::string &Result,
-                          bool CanHaveLeadingDot = true);
+                          bool CanHaveLeadingDot = true,
+                          bool ParseParams = true);
 
 /// "Partial" demangler. This supports demangling a string into an AST
 /// (typically an intermediate stage in itaniumDemangle) and querying certain
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index e0ff035d47cf..06956f47c1f0 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -2793,7 +2793,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser {
   Node *parseClassEnumType();
   Node *parseQualifiedType();
 
-  Node *parseEncoding();
+  Node *parseEncoding(bool ParseParams = true);
   bool parseCallOffset();
   Node *parseSpecialName();
 
@@ -2910,7 +2910,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser {
   Node *parseDestructorName();
 
   /// Top-level entry point into the parser.
-  Node *parse();
+  Node *parse(bool ParseParams = true);
 };
 
 const char* parse_discriminator(const char* first, const char* last);
@@ -5404,7 +5404,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSpecialName() {
 //            ::= <data name>
 //            ::= <special-name>
 template <typename Derived, typename Alloc>
-Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() {
+Node *AbstractManglingParser<Derived, Alloc>::parseEncoding(bool ParseParams) {
   // The template parameters of an encoding are unrelated to those of the
   // enclosing context.
   SaveTemplateParams SaveTemplateParamsScope(this);
@@ -5430,6 +5430,16 @@ Node *AbstractManglingParser<Derived, Alloc>::parseEncoding() {
   if (IsEndOfEncoding())
     return Name;
 
+  // ParseParams may be false at the top level only, when called from parse().
+  // For example in the mangled name _Z3fooILZ3BarEET_f, ParseParams may be
+  // false when demangling 3fooILZ3BarEET_f but is always true when demangling
+  // 3Bar.
+  if (!ParseParams) {
+    while (consume())
+      ;
+    return Name;
+  }
+
   Node *Attrs = nullptr;
   if (consumeIf("Ua9enable_ifI")) {
     size_t BeforeArgs = Names.size();
@@ -5894,9 +5904,9 @@ AbstractManglingParser<Derived, Alloc>::parseTemplateArgs(bool TagTemplates) {
 // extension      ::= ___Z <encoding> _block_invoke<decimal-digit>+
 // extension      ::= ___Z <encoding> _block_invoke_<decimal-digit>+
 template <typename Derived, typename Alloc>
-Node *AbstractManglingParser<Derived, Alloc>::parse() {
+Node *AbstractManglingParser<Derived, Alloc>::parse(bool ParseParams) {
   if (consumeIf("_Z") || consumeIf("__Z")) {
-    Node *Encoding = getDerived().parseEncoding();
+    Node *Encoding = getDerived().parseEncoding(ParseParams);
     if (Encoding == nullptr)
       return nullptr;
     if (look() == '.') {
@@ -5910,7 +5920,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parse() {
   }
 
   if (consumeIf("___Z") || consumeIf("____Z")) {
-    Node *Encoding = getDerived().parseEncoding();
+    Node *Encoding = getDerived().parseEncoding(ParseParams);
     if (Encoding == nullptr || !consumeIf("_block_invoke"))
       return nullptr;
     bool RequireNumber = consumeIf('_');
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 8a019492c12d..f4d2f56c34d9 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
@@ -1923,6 +1924,10 @@ void visitExistingEdges(LinkGraph &G, VisitorTs &&...Vs) {
 Expected<std::unique_ptr<LinkGraph>>
 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer);
 
+/// Create a \c LinkGraph defining the given absolute symbols.
+std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
+                                                    orc::SymbolMap Symbols);
+
 /// Link the given graph.
 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx);
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index ba164c6b629e..6a9bcf712169 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -1210,14 +1210,13 @@ private:
     SymbolTableEntry() = default;
     SymbolTableEntry(JITSymbolFlags Flags)
         : Flags(Flags), State(static_cast<uint8_t>(SymbolState::NeverSearched)),
-          MaterializerAttached(false), PendingRemoval(false) {}
+          MaterializerAttached(false) {}
 
     ExecutorAddr getAddress() const { return Addr; }
     JITSymbolFlags getFlags() const { return Flags; }
     SymbolState getState() const { return static_cast<SymbolState>(State); }
 
     bool hasMaterializerAttached() const { return MaterializerAttached; }
-    bool isPendingRemoval() const { return PendingRemoval; }
 
     void setAddress(ExecutorAddr Addr) { this->Addr = Addr; }
     void setFlags(JITSymbolFlags Flags) { this->Flags = Flags; }
@@ -1231,18 +1230,13 @@ private:
       this->MaterializerAttached = MaterializerAttached;
     }
 
-    void setPendingRemoval(bool PendingRemoval) {
-      this->PendingRemoval = PendingRemoval;
-    }
-
     ExecutorSymbolDef getSymbol() const { return {Addr, Flags}; }
 
   private:
     ExecutorAddr Addr;
     JITSymbolFlags Flags;
-    uint8_t State : 6;
+    uint8_t State : 7;
     uint8_t MaterializerAttached : 1;
-    uint8_t PendingRemoval : 1;
   };
 
   using SymbolTable = DenseMap<SymbolStringPtr, SymbolTableEntry>;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
index 63797edec89e..e56afe4fe656 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
@@ -25,6 +25,7 @@ class ExecutorProcessControl;
 class EPCDynamicLibrarySearchGenerator : public DefinitionGenerator {
 public:
   using SymbolPredicate = unique_function<bool(const SymbolStringPtr &)>;
+  using AddAbsoluteSymbolsFn = unique_function<Error(JITDylib &, SymbolMap)>;
 
   /// Create a DynamicLibrarySearchGenerator that searches for symbols in the
   /// library with the given handle.
@@ -32,24 +33,31 @@ public:
   /// If the Allow predicate is given then only symbols matching the predicate
   /// will be searched for. If the predicate is not given then all symbols will
   /// be searched for.
-  EPCDynamicLibrarySearchGenerator(ExecutionSession &ES,
-                                   tpctypes::DylibHandle H,
-                                   SymbolPredicate Allow = SymbolPredicate())
-      : EPC(ES.getExecutorProcessControl()), H(H), Allow(std::move(Allow)) {}
+  ///
+  /// If \p AddAbsoluteSymbols is provided, it is used to add the symbols to the
+  /// \c JITDylib; otherwise it uses JD.define(absoluteSymbols(...)).
+  EPCDynamicLibrarySearchGenerator(
+      ExecutionSession &ES, tpctypes::DylibHandle H,
+      SymbolPredicate Allow = SymbolPredicate(),
+      AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr)
+      : EPC(ES.getExecutorProcessControl()), H(H), Allow(std::move(Allow)),
+        AddAbsoluteSymbols(std::move(AddAbsoluteSymbols)) {}
 
   /// Permanently loads the library at the given path and, on success, returns
   /// a DynamicLibrarySearchGenerator that will search it for symbol definitions
   /// in the library. On failure returns the reason the library failed to load.
   static Expected<std::unique_ptr<EPCDynamicLibrarySearchGenerator>>
   Load(ExecutionSession &ES, const char *LibraryPath,
-       SymbolPredicate Allow = SymbolPredicate());
+       SymbolPredicate Allow = SymbolPredicate(),
+       AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr);
 
   /// Creates a EPCDynamicLibrarySearchGenerator that searches for symbols in
   /// the target process.
   static Expected<std::unique_ptr<EPCDynamicLibrarySearchGenerator>>
   GetForTargetProcess(ExecutionSession &ES,
-                      SymbolPredicate Allow = SymbolPredicate()) {
-    return Load(ES, nullptr, std::move(Allow));
+                      SymbolPredicate Allow = SymbolPredicate(),
+                      AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr) {
+    return Load(ES, nullptr, std::move(Allow), std::move(AddAbsoluteSymbols));
   }
 
   Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD,
@@ -60,6 +68,7 @@ private:
   ExecutorProcessControl &EPC;
   tpctypes::DylibHandle H;
   SymbolPredicate Allow;
+  AddAbsoluteSymbolsFn AddAbsoluteSymbols;
 };
 
 } // end namespace orc
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
index 02e580c86f54..6ee2deef04d0 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
@@ -19,6 +19,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_EPCGENERICDYLIBMANAGER_H
 
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
 
 namespace llvm {
@@ -49,11 +50,11 @@ public:
   Expected<tpctypes::DylibHandle> open(StringRef Path, uint64_t Mode);
 
   /// Looks up symbols within the given dylib.
-  Expected<std::vector<ExecutorAddr>> lookup(tpctypes::DylibHandle H,
-                                             const SymbolLookupSet &Lookup);
+  Expected<std::vector<ExecutorSymbolDef>>
+  lookup(tpctypes::DylibHandle H, const SymbolLookupSet &Lookup);
 
   /// Looks up symbols within the given dylib.
-  Expected<std::vector<ExecutorAddr>>
+  Expected<std::vector<ExecutorSymbolDef>>
   lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &Lookup);
 
 private:
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index 6a43376a5bd9..f7c286bec778 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -216,6 +216,7 @@ private:
 class DynamicLibrarySearchGenerator : public DefinitionGenerator {
 public:
   using SymbolPredicate = std::function<bool(const SymbolStringPtr &)>;
+  using AddAbsoluteSymbolsFn = unique_function<Error(JITDylib &, SymbolMap)>;
 
   /// Create a DynamicLibrarySearchGenerator that searches for symbols in the
   /// given sys::DynamicLibrary.
@@ -223,22 +224,30 @@ public:
   /// If the Allow predicate is given then only symbols matching the predicate
   /// will be searched for. If the predicate is not given then all symbols will
   /// be searched for.
-  DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, char GlobalPrefix,
-                                SymbolPredicate Allow = SymbolPredicate());
+  ///
+  /// If \p AddAbsoluteSymbols is provided, it is used to add the symbols to the
+  /// \c JITDylib; otherwise it uses JD.define(absoluteSymbols(...)).
+  DynamicLibrarySearchGenerator(
+      sys::DynamicLibrary Dylib, char GlobalPrefix,
+      SymbolPredicate Allow = SymbolPredicate(),
+      AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr);
 
   /// Permanently loads the library at the given path and, on success, returns
   /// a DynamicLibrarySearchGenerator that will search it for symbol definitions
   /// in the library. On failure returns the reason the library failed to load.
   static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>>
   Load(const char *FileName, char GlobalPrefix,
-       SymbolPredicate Allow = SymbolPredicate());
+       SymbolPredicate Allow = SymbolPredicate(),
+       AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr);
 
   /// Creates a DynamicLibrarySearchGenerator that searches for symbols in
   /// the current process.
   static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>>
   GetForCurrentProcess(char GlobalPrefix,
-                       SymbolPredicate Allow = SymbolPredicate()) {
-    return Load(nullptr, GlobalPrefix, std::move(Allow));
+                       SymbolPredicate Allow = SymbolPredicate(),
+                       AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr) {
+    return Load(nullptr, GlobalPrefix, std::move(Allow),
+                std::move(AddAbsoluteSymbols));
   }
 
   Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD,
@@ -248,6 +257,7 @@ public:
 private:
   sys::DynamicLibrary Dylib;
   SymbolPredicate Allow;
+  AddAbsoluteSymbolsFn AddAbsoluteSymbols;
   char GlobalPrefix;
 };
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h
index 5c58a7255ebd..68ccdf83bd12 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
 
 namespace llvm {
 namespace orc {
@@ -48,6 +49,63 @@ private:
   JITSymbolFlags Flags;
 };
 
+namespace shared {
+
+using SPSJITSymbolFlags =
+    SPSTuple<JITSymbolFlags::UnderlyingType, JITSymbolFlags::TargetFlagsType>;
+
+/// SPS serializatior for JITSymbolFlags.
+template <> class SPSSerializationTraits<SPSJITSymbolFlags, JITSymbolFlags> {
+  using FlagsArgList = SPSJITSymbolFlags::AsArgList;
+
+public:
+  static size_t size(const JITSymbolFlags &F) {
+    return FlagsArgList::size(F.getRawFlagsValue(), F.getTargetFlags());
+  }
+
+  static bool serialize(SPSOutputBuffer &BOB, const JITSymbolFlags &F) {
+    return FlagsArgList::serialize(BOB, F.getRawFlagsValue(),
+                                   F.getTargetFlags());
+  }
+
+  static bool deserialize(SPSInputBuffer &BIB, JITSymbolFlags &F) {
+    JITSymbolFlags::UnderlyingType RawFlags;
+    JITSymbolFlags::TargetFlagsType TargetFlags;
+    if (!FlagsArgList::deserialize(BIB, RawFlags, TargetFlags))
+      return false;
+    F = JITSymbolFlags{static_cast<JITSymbolFlags::FlagNames>(RawFlags),
+                       TargetFlags};
+    return true;
+  }
+};
+
+using SPSExecutorSymbolDef = SPSTuple<SPSExecutorAddr, SPSJITSymbolFlags>;
+
+/// SPS serializatior for ExecutorSymbolDef.
+template <>
+class SPSSerializationTraits<SPSExecutorSymbolDef, ExecutorSymbolDef> {
+  using DefArgList = SPSExecutorSymbolDef::AsArgList;
+
+public:
+  static size_t size(const ExecutorSymbolDef &ESD) {
+    return DefArgList::size(ESD.getAddress(), ESD.getFlags());
+  }
+
+  static bool serialize(SPSOutputBuffer &BOB, const ExecutorSymbolDef &ESD) {
+    return DefArgList::serialize(BOB, ESD.getAddress(), ESD.getFlags());
+  }
+
+  static bool deserialize(SPSInputBuffer &BIB, ExecutorSymbolDef &ESD) {
+    ExecutorAddr Addr;
+    JITSymbolFlags Flags;
+    if (!DefArgList::deserialize(BIB, Addr, Flags))
+      return false;
+    ESD = ExecutorSymbolDef{Addr, Flags};
+    return true;
+  }
+};
+
+} // End namespace shared.
 } // End namespace orc.
 } // End namespace llvm.
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index 8e336ca03eaf..0c549bcbf013 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_SHARED_ORCRTBRIDGE_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 
@@ -54,7 +55,7 @@ using SPSSimpleExecutorDylibManagerOpenSignature =
                                                  shared::SPSString, uint64_t);
 
 using SPSSimpleExecutorDylibManagerLookupSignature =
-    shared::SPSExpected<shared::SPSSequence<shared::SPSExecutorAddr>>(
+    shared::SPSExpected<shared::SPSSequence<shared::SPSExecutorSymbolDef>>(
         shared::SPSExecutorAddr, shared::SPSExecutorAddr,
         shared::SPSRemoteSymbolLookupSet);
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h
index 7322674559c9..e91d8d926d88 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
@@ -113,7 +114,7 @@ struct PointerWrite {
 /// A handle used to represent a loaded dylib in the target process.
 using DylibHandle = ExecutorAddr;
 
-using LookupResult = std::vector<ExecutorAddr>;
+using LookupResult = std::vector<ExecutorSymbolDef>;
 
 } // end namespace tpctypes
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
index 99175d796974..9f91a64e95ce 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h
@@ -16,6 +16,32 @@
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
 #include <cstdint>
 
+// Keep in sync with gdb/gdb/jit.h
+extern "C" {
+
+typedef enum {
+  JIT_NOACTION = 0,
+  JIT_REGISTER_FN,
+  JIT_UNREGISTER_FN
+} jit_actions_t;
+
+struct jit_code_entry {
+  struct jit_code_entry *next_entry;
+  struct jit_code_entry *prev_entry;
+  const char *symfile_addr;
+  uint64_t symfile_size;
+};
+
+struct jit_descriptor {
+  uint32_t version;
+  // This should be jit_actions_t, but we want to be specific about the
+  // bit-width.
+  uint32_t action_flag;
+  struct jit_code_entry *relevant_entry;
+  struct jit_code_entry *first_entry;
+};
+}
+
 extern "C" llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size);
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
index fd4504cfb7fb..00fd84e3ec14 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
@@ -18,6 +18,7 @@
 
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
@@ -37,8 +38,8 @@ public:
   virtual ~SimpleExecutorDylibManager();
 
   Expected<tpctypes::DylibHandle> open(const std::string &Path, uint64_t Mode);
-  Expected<std::vector<ExecutorAddr>> lookup(tpctypes::DylibHandle H,
-                                             const RemoteSymbolLookupSet &L);
+  Expected<std::vector<ExecutorSymbolDef>>
+  lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &L);
 
   Error shutdown() override;
   void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override;
diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h
index 8784a425d284..42db4c4ea3a5 100644
--- a/llvm/include/llvm/IR/Dominators.h
+++ b/llvm/include/llvm/IR/Dominators.h
@@ -293,11 +293,14 @@ public:
   explicit DominatorTreePrinterPass(raw_ostream &OS);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 
 /// Verifier pass for the \c DominatorTree.
 struct DominatorTreeVerifierPass : PassInfoMixin<DominatorTreeVerifierPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Enables verification of dominator trees.
diff --git a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h
index f3272327c3f8..1092b636e023 100644
--- a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h
+++ b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/PointerUnion.h"
+#include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/User.h"
@@ -30,7 +31,39 @@ template <typename ItTy = User::const_op_iterator>
 class generic_gep_type_iterator {
 
   ItTy OpIt;
-  PointerUnion<StructType *, Type *> CurTy;
+  // We use two different mechanisms to store the type a GEP index applies to.
+  // In some cases, we need to know the outer aggregate type the index is
+  // applied within, e.g. a struct. In such cases, we store the aggregate type
+  // in the iterator, and derive the element type on the fly.
+  //
+  // However, this is not always possible, because for the outermost index there
+  // is no containing type. In such cases, or if the containing type is not
+  // relevant, e.g. for arrays, the element type is stored as Type* in CurTy.
+  //
+  // If CurTy contains a Type* value, this does not imply anything about the
+  // type itself, because it is the element type and not the outer type.
+  // In particular, Type* can be a struct type.
+  //
+  // Consider this example:
+  //
+  //    %my.struct = type { i32, [ 4 x float ] }
+  //    [...]
+  //    %gep = getelementptr %my.struct, ptr %ptr, i32 10, i32 1, 32 3
+  //
+  // Iterating over the indices of this GEP, CurTy will contain the following
+  // values:
+  //    * i32 10: The outer index always operates on the GEP value type.
+  //              CurTy contains a Type*       pointing at `%my.struct`.
+  //    * i32 1:  This index is within a struct.
+  //              CurTy contains a StructType* pointing at `%my.struct`.
+  //    * i32 3:  This index is within an array. We reuse the "flat" indexing
+  //              for arrays which is also used in the top level GEP index.
+  //              CurTy contains a Type*       pointing at `float`.
+  //
+  // Vectors are handled separately because the layout of vectors is different
+  // for overaligned elements: Vectors are always bit-packed, whereas arrays
+  // respect ABI alignment of the elements.
+  PointerUnion<StructType *, VectorType *, Type *> CurTy;
 
   generic_gep_type_iterator() = default;
 
@@ -69,6 +102,8 @@ public:
   Type *getIndexedType() const {
     if (auto *T = dyn_cast_if_present<Type *>(CurTy))
       return T;
+    if (auto *VT = dyn_cast_if_present<VectorType *>(CurTy))
+      return VT->getElementType();
     return cast<StructType *>(CurTy)->getTypeAtIndex(getOperand());
   }
 
@@ -79,7 +114,7 @@ public:
     if (auto *ATy = dyn_cast<ArrayType>(Ty))
       CurTy = ATy->getElementType();
     else if (auto *VTy = dyn_cast<VectorType>(Ty))
-      CurTy = VTy->getElementType();
+      CurTy = VTy;
     else
       CurTy = dyn_cast<StructType>(Ty);
     ++OpIt;
@@ -108,7 +143,23 @@ public:
   // that.
 
   bool isStruct() const { return isa<StructType *>(CurTy); }
-  bool isSequential() const { return isa<Type *>(CurTy); }
+  bool isVector() const { return isa<VectorType *>(CurTy); }
+  bool isSequential() const { return !isStruct(); }
+
+  // For sequential GEP indices (all except those into structs), the index value
+  // can be translated into a byte offset by multiplying with an element stride.
+  // This function returns this stride, which both depends on the element type,
+  // and the containing aggregate type, as vectors always tightly bit-pack their
+  // elements.
+  TypeSize getSequentialElementStride(const DataLayout &DL) const {
+    assert(isSequential());
+    Type *ElemTy = getIndexedType();
+    if (isVector()) {
+      assert(DL.typeSizeEqualsStoreSize(ElemTy) && "Not byte-addressable");
+      return DL.getTypeStoreSize(ElemTy);
+    }
+    return DL.getTypeAllocSize(ElemTy);
+  }
 
   StructType *getStructType() const { return cast<StructType *>(CurTy); }
 
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 531b11123545..e5596258847f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -837,7 +837,7 @@ class AMDGPUImageDimIntrinsicEval<AMDGPUDimProfile P_> {
 // All dimension-aware intrinsics are derived from this class.
 class AMDGPUImageDimIntrinsic<AMDGPUDimProfile P_,
                               list<IntrinsicProperty> props,
-                              list<SDNodeProperty> sdnodeprops> : DefaultAttrsIntrinsic<
+                              list<SDNodeProperty> sdnodeprops> : Intrinsic<
     P_.RetTypes,        // vdata(VGPR) -- for load/atomic-with-return
     !listconcat(
       !foreach(arg, P_.DataArgs, arg.Type),      // vdata(VGPR) -- for store/atomic
@@ -851,11 +851,12 @@ class AMDGPUImageDimIntrinsic<AMDGPUDimProfile P_,
                                                  //   gfx12+ imm: bits [0-2] = th, bits [3-4] = scope)
                                                  // TODO-GFX12: Update all other cachepolicy descriptions.
 
-     !listconcat(props,
+     !listconcat(props, [IntrNoCallback, IntrNoFree, IntrWillReturn],
           !if(P_.IsAtomic, [], [ImmArg<ArgIndex<AMDGPUImageDimIntrinsicEval<P_>.DmaskArgIndex>>]),
           !if(P_.IsSample, [ImmArg<ArgIndex<AMDGPUImageDimIntrinsicEval<P_>.UnormArgIndex>>], []),
           [ImmArg<ArgIndex<AMDGPUImageDimIntrinsicEval<P_>.TexFailCtrlArgIndex>>,
-           ImmArg<ArgIndex<AMDGPUImageDimIntrinsicEval<P_>.CachePolicyArgIndex>>]),
+           ImmArg<ArgIndex<AMDGPUImageDimIntrinsicEval<P_>.CachePolicyArgIndex>>],
+          !if(P_.IsAtomic, [], [IntrNoSync])),
 
 
       "", sdnodeprops>,
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 736be8ca3212..ea0074d22a44 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -28,6 +28,7 @@ let TargetPrefix = "spv" in {
   def int_spv_insertelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_anyint_ty]>;
   def int_spv_const_composite : Intrinsic<[llvm_i32_ty], [llvm_vararg_ty]>;
   def int_spv_bitcast : Intrinsic<[llvm_any_ty], [llvm_any_ty]>;
+  def int_spv_ptrcast : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
   def int_spv_switch : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>;
   def int_spv_cmpxchg : Intrinsic<[llvm_i32_ty], [llvm_any_ty, llvm_vararg_ty]>;
   def int_spv_unreachable : Intrinsic<[], []>;
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 48afdb867ba6..447ac0f2aa61 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1270,7 +1270,7 @@ inline DisjointOr_match<LHS, RHS, true> m_c_DisjointOr(const LHS &L,
   return DisjointOr_match<LHS, RHS, true>(L, R);
 }
 
-/// Match either "and" or "or disjoint".
+/// Match either "add" or "or disjoint".
 template <typename LHS, typename RHS>
 inline match_combine_or<BinaryOp_match<LHS, RHS, Instruction::Add>,
                         DisjointOr_match<LHS, RHS>>
diff --git a/llvm/include/llvm/IR/SafepointIRVerifier.h b/llvm/include/llvm/IR/SafepointIRVerifier.h
index 246d236adb38..2ee998e4c68f 100644
--- a/llvm/include/llvm/IR/SafepointIRVerifier.h
+++ b/llvm/include/llvm/IR/SafepointIRVerifier.h
@@ -40,6 +40,8 @@ public:
   explicit SafepointIRVerifierPass() = default;
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
 };
 }
 
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 46b1e95c3c15..3db639a68724 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -54,7 +54,7 @@ void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
 void initializeAssumptionCacheTrackerPass(PassRegistry&);
 void initializeAtomicExpandPass(PassRegistry&);
 void initializeBasicBlockPathCloningPass(PassRegistry &);
-void initializeBasicBlockSectionsProfileReaderPass(PassRegistry &);
+void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &);
 void initializeBasicBlockSectionsPass(PassRegistry &);
 void initializeBarrierNoopPass(PassRegistry&);
 void initializeBasicAAWrapperPassPass(PassRegistry&);
@@ -75,7 +75,7 @@ void initializeCallGraphDOTPrinterPass(PassRegistry&);
 void initializeCallGraphViewerPass(PassRegistry&);
 void initializeCallGraphWrapperPassPass(PassRegistry&);
 void initializeCheckDebugMachineModulePass(PassRegistry &);
-void initializeCodeGenPreparePass(PassRegistry&);
+void initializeCodeGenPrepareLegacyPassPass(PassRegistry &);
 void initializeComplexDeinterleavingLegacyPassPass(PassRegistry&);
 void initializeConstantHoistingLegacyPassPass(PassRegistry&);
 void initializeCycleInfoWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index be85c4098347..1050f24161fb 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -404,7 +404,9 @@ private:
   };
 
   // Global mapping from mangled symbol names to resolutions.
-  StringMap<GlobalResolution> GlobalResolutions;
+  // Make this an optional to guard against accessing after it has been reset
+  // (to reduce memory after we're done with it).
+  std::optional<StringMap<GlobalResolution>> GlobalResolutions;
 
   void addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
                             ArrayRef<SymbolResolution> Res, unsigned Partition,
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 7a21876e565a..fe7fedad18bc 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -113,7 +113,7 @@ namespace {
       (void) llvm::createTailCallEliminationPass();
       (void)llvm::createTLSVariableHoistPass();
       (void) llvm::createConstantHoistingPass();
-      (void) llvm::createCodeGenPreparePass();
+      (void)llvm::createCodeGenPrepareLegacyPass();
       (void) llvm::createEarlyCSEPass();
       (void) llvm::createGVNPass();
       (void) llvm::createPostDomTree();
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index 8931e8cab2fa..01a64fb425a9 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -198,9 +198,9 @@ public:
 
   // Defined by linker relaxation targets to possibly emit LEB128 relocations
   // and set Value at the relocated location.
-  virtual bool relaxLEB128(MCLEBFragment &LF, MCAsmLayout &Layout,
-                           int64_t &Value) const {
-    return false;
+  virtual std::pair<bool, bool>
+  relaxLEB128(MCLEBFragment &LF, MCAsmLayout &Layout, int64_t &Value) const {
+    return std::make_pair(false, false);
   }
 
   /// @}
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index 99477644de4d..86f42654f8af 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -454,6 +454,7 @@ public:
   uint8_t getBytesInAddress() const override;
   StringRef getFileFormatName() const override;
   Triple::ArchType getArch() const override;
+  Triple::OSType getOS() const override;
   Expected<uint64_t> getStartAddress() const override;
 
   unsigned getPlatformFlags() const override { return EF.getHeader().e_flags; }
@@ -1349,6 +1350,12 @@ template <class ELFT> Triple::ArchType ELFObjectFile<ELFT>::getArch() const {
     return Triple::UnknownArch;
   }
 
+  case ELF::EM_CUDA: {
+    if (EF.getHeader().e_ident[ELF::EI_CLASS] == ELF::ELFCLASS32)
+      return Triple::nvptx;
+    return Triple::nvptx64;
+  }
+
   case ELF::EM_BPF:
     return IsLittleEndian ? Triple::bpfel : Triple::bpfeb;
 
@@ -1375,6 +1382,35 @@ template <class ELFT> Triple::ArchType ELFObjectFile<ELFT>::getArch() const {
   }
 }
 
+template <class ELFT> Triple::OSType ELFObjectFile<ELFT>::getOS() const {
+  switch (EF.getHeader().e_ident[ELF::EI_OSABI]) {
+  case ELF::ELFOSABI_NETBSD:
+    return Triple::NetBSD;
+  case ELF::ELFOSABI_LINUX:
+    return Triple::Linux;
+  case ELF::ELFOSABI_HURD:
+    return Triple::Hurd;
+  case ELF::ELFOSABI_SOLARIS:
+    return Triple::Solaris;
+  case ELF::ELFOSABI_AIX:
+    return Triple::AIX;
+  case ELF::ELFOSABI_FREEBSD:
+    return Triple::FreeBSD;
+  case ELF::ELFOSABI_OPENBSD:
+    return Triple::OpenBSD;
+  case ELF::ELFOSABI_CUDA:
+    return Triple::CUDA;
+  case ELF::ELFOSABI_AMDGPU_HSA:
+    return Triple::AMDHSA;
+  case ELF::ELFOSABI_AMDGPU_PAL:
+    return Triple::AMDPAL;
+  case ELF::ELFOSABI_AMDGPU_MESA3D:
+    return Triple::Mesa3D;
+  default:
+    return Triple::UnknownOS;
+  }
+}
+
 template <class ELFT>
 Expected<uint64_t> ELFObjectFile<ELFT>::getStartAddress() const {
   return EF.getHeader().e_entry;
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index d3351a2d1650..956f7811dd6c 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -885,6 +885,9 @@ struct PGOAnalysisMap {
     bool BBFreq : 1;
     bool BrProb : 1;
 
+    // True if at least one feature is enabled
+    bool anyEnabled() const { return FuncEntryCount || BBFreq || BrProb; }
+
     // Encodes to minimum bit width representation.
     uint8_t encode() const {
       return (static_cast<uint8_t>(FuncEntryCount) << 0) |
diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h
index c254fc2ccfde..8c868c7643ed 100644
--- a/llvm/include/llvm/Object/ObjectFile.h
+++ b/llvm/include/llvm/Object/ObjectFile.h
@@ -337,6 +337,7 @@ public:
 
   virtual StringRef getFileFormatName() const = 0;
   virtual Triple::ArchType getArch() const = 0;
+  virtual Triple::OSType getOS() const { return Triple::UnknownOS; }
   virtual Expected<SubtargetFeatures> getFeatures() const = 0;
   virtual std::optional<StringRef> tryGetCPUName() const {
     return std::nullopt;
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 2ec36cad244f..8c6a44876d54 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -65,7 +65,7 @@ private:
   bool shouldPrintBeforePass(StringRef PassID);
   bool shouldPrintAfterPass(StringRef PassID);
   bool shouldPrintPassNumbers();
-  bool shouldPrintAtPassNumber();
+  bool shouldPrintBeforePassNumber();
 
   void pushPassRunDescriptor(StringRef PassID, Any IR,
                              std::string &DumpIRFilename);
diff --git a/llvm/include/llvm/Support/RISCVISAInfo.h b/llvm/include/llvm/Support/RISCVISAInfo.h
index 09c4edd6df60..c539448683d3 100644
--- a/llvm/include/llvm/Support/RISCVISAInfo.h
+++ b/llvm/include/llvm/Support/RISCVISAInfo.h
@@ -68,9 +68,8 @@ public:
   parseFeatures(unsigned XLen, const std::vector<std::string> &Features);
 
   /// Convert RISC-V ISA info to a feature vector.
-  void toFeatures(std::vector<StringRef> &Features,
-                  llvm::function_ref<StringRef(const Twine &)> StrAlloc,
-                  bool AddAllExtensions) const;
+  std::vector<std::string> toFeatures(bool AddAllExtensions = false,
+                                      bool IgnoreUnknown = true) const;
 
   const OrderedExtensionMap &getExtensions() const { return Exts; };
 
@@ -83,7 +82,6 @@ public:
 
   bool hasExtension(StringRef Ext) const;
   std::string toString() const;
-  std::vector<std::string> toFeatureVector() const;
   StringRef computeDefaultABI() const;
 
   static bool isSupportedExtensionFeature(StringRef Ext);
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index f28c1edc3d95..5e704f0b9a75 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -244,6 +244,7 @@ def : GINodeEquiv<G_ATOMICRMW_FMIN, atomic_load_fmin>;
 def : GINodeEquiv<G_ATOMICRMW_UINC_WRAP, atomic_load_uinc_wrap>;
 def : GINodeEquiv<G_ATOMICRMW_UDEC_WRAP, atomic_load_udec_wrap>;
 def : GINodeEquiv<G_FENCE, atomic_fence>;
+def : GINodeEquiv<G_PREFETCH, prefetch>;
 
 // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern.
 // Should be used on defs that subclass GIComplexOperandMatcher<>.
diff --git a/llvm/include/llvm/Target/TargetInstrPredicate.td b/llvm/include/llvm/Target/TargetInstrPredicate.td
index 9f2cde9d9230..82c4c7b23a49 100644
--- a/llvm/include/llvm/Target/TargetInstrPredicate.td
+++ b/llvm/include/llvm/Target/TargetInstrPredicate.td
@@ -95,6 +95,12 @@ class MCOperandPredicate<int Index> : MCInstPredicate {
 // Return true if machine operand at position `Index` is a register operand.
 class CheckIsRegOperand<int Index> : MCOperandPredicate<Index>;
 
+// Return true if machine operand at position `Index` is a virtual register operand.
+class CheckIsVRegOperand<int Index> : MCOperandPredicate<Index>;
+
+// Return true if machine operand at position `Index` is not a virtual register operand.
+class CheckIsNotVRegOperand<int Index> : CheckNot<CheckIsVRegOperand<Index>>;
+
 // Return true if machine operand at position `Index` is an immediate operand.
 class CheckIsImmOperand<int Index> : MCOperandPredicate<Index>;
 
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 4c29f25bedf4..1fe47dec70b1 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -362,7 +362,9 @@ public:
   virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const;
 
   /// Allow the target to modify the pass pipeline.
-  virtual void registerPassBuilderCallbacks(PassBuilder &) {}
+  // TODO: Populate all pass names by using <Target>PassRegistry.def.
+  virtual void registerPassBuilderCallbacks(PassBuilder &,
+                                            bool PopulateClassToPassNames) {}
 
   /// Allow the target to register alias analyses with the AAManager for use
   /// with the new pass manager. Only affects the "default" AAManager.
diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td
index 949baa5d2105..2016d452afb6 100644
--- a/llvm/include/llvm/Target/TargetSchedule.td
+++ b/llvm/include/llvm/Target/TargetSchedule.td
@@ -584,3 +584,116 @@ class MemoryQueue<ProcResourceKind PR> {
 
 class LoadQueue<ProcResourceKind LDQueue> : MemoryQueue<LDQueue>;
 class StoreQueue<ProcResourceKind STQueue> : MemoryQueue<STQueue>;
+
+// The target instruction that FusionPredicate will be evaluated on.
+class FusionTarget;
+def first_fusion_target : FusionTarget;
+def second_fusion_target : FusionTarget;
+def both_fusion_target : FusionTarget;
+
+// Base class of FusionPredicate, etc. The avaliable variables are:
+// * const TargetInstrInfo &TII
+// * const TargetSubtargetInfo &STI
+// * const MachineRegisterInfo &MRI
+// * const MachineInstr *FirstMI
+// * const MachineInstr &SecondMI
+class FusionPredicate<FusionTarget target> {
+  FusionTarget Target = target;
+}
+class FirstFusionPredicate: FusionPredicate<first_fusion_target>;
+class SecondFusionPredicate: FusionPredicate<second_fusion_target>;
+class BothFusionPredicate: FusionPredicate<both_fusion_target>;
+
+// FusionPredicate with raw code predicate.
+class FusionPredicateWithCode<code pred> : FusionPredicate<both_fusion_target> {
+  code Predicate = pred;
+}
+
+// FusionPredicate with MCInstPredicate.
+class FusionPredicateWithMCInstPredicate<FusionTarget target, MCInstPredicate pred>
+  : FusionPredicate<target> {
+  MCInstPredicate Predicate = pred;
+}
+class FirstFusionPredicateWithMCInstPredicate<MCInstPredicate pred>
+  : FusionPredicateWithMCInstPredicate<first_fusion_target, pred>;
+class SecondFusionPredicateWithMCInstPredicate<MCInstPredicate pred>
+  : FusionPredicateWithMCInstPredicate<second_fusion_target, pred>;
+// The pred will be applied on both firstMI and secondMI.
+class BothFusionPredicateWithMCInstPredicate<MCInstPredicate pred>
+  : FusionPredicateWithMCInstPredicate<second_fusion_target, pred>;
+
+// Tie firstOpIdx and secondOpIdx. The operand of `FirstMI` at position
+// `firstOpIdx` should be the same as the operand of `SecondMI` at position
+// `secondOpIdx`.
+class TieReg<int firstOpIdx, int secondOpIdx> : BothFusionPredicate {
+  int FirstOpIdx = firstOpIdx;
+  int SecondOpIdx = secondOpIdx;
+}
+
+// A predicate for wildcard. The generated code will be like:
+// ```
+// if (!FirstMI)
+//   return ReturnValue;
+// ```
+class WildcardPred<bit ret> : FirstFusionPredicate {
+  bit ReturnValue = ret;
+}
+def WildcardFalse : WildcardPred<0>;
+def WildcardTrue : WildcardPred<1>;
+
+// Indicates that the destination register of `FirstMI` should have one use if
+// it is a virtual register.
+class OneUsePred : FirstFusionPredicate;
+def OneUse : OneUsePred;
+
+// Handled by MacroFusionPredicatorEmitter backend.
+// The generated predicator will be like:
+// ```
+// bool isNAME(const TargetInstrInfo &TII,
+//             const TargetSubtargetInfo &STI,
+//             const MachineInstr *FirstMI,
+//             const MachineInstr &SecondMI) {
+//   auto &MRI = SecondMI.getMF()->getRegInfo();
+//   /* Predicates */
+//   return true;
+// }
+// ```
+class Fusion<list<FusionPredicate> predicates> {
+  list<FusionPredicate> Predicates = predicates;
+}
+
+// The generated predicator will be like:
+// ```
+// bool isNAME(const TargetInstrInfo &TII,
+//             const TargetSubtargetInfo &STI,
+//             const MachineInstr *FirstMI,
+//             const MachineInstr &SecondMI) {
+//   auto &MRI = SecondMI.getMF()->getRegInfo();
+//   /* Prolog */
+//   /* Predicate for `SecondMI` */
+//   /* Wildcard */
+//   /* Predicate for `FirstMI` */
+//   /* Check One Use */
+//   /* Tie registers */
+//   /* Epilog */
+//   return true;
+// }
+// ```
+class SimpleFusion<MCInstPredicate firstPred, MCInstPredicate secondPred,
+                   list<FusionPredicate> prolog = [],
+                   list<FusionPredicate> epilog = []>
+  : Fusion<!listconcat(
+                  prolog,
+                  [
+                    SecondFusionPredicateWithMCInstPredicate<secondPred>,
+                    WildcardTrue,
+                    FirstFusionPredicateWithMCInstPredicate<firstPred>,
+                    SecondFusionPredicateWithMCInstPredicate<
+                      CheckAny<[
+                        CheckIsVRegOperand<0>,
+                        CheckSameRegOperand<0, 1>
+                      ]>>,
+                    OneUse,
+                    TieReg<0, 1>,
+                  ],
+                  epilog)>;
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index d4fc9d8a96db..22360353790d 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -614,6 +614,12 @@ def strict_sint_to_fp : SDNode<"ISD::STRICT_SINT_TO_FP",
                                SDTIntToFPOp, [SDNPHasChain]>;
 def strict_uint_to_fp : SDNode<"ISD::STRICT_UINT_TO_FP",
                                SDTIntToFPOp, [SDNPHasChain]>;
+
+def strict_f16_to_fp  : SDNode<"ISD::STRICT_FP16_TO_FP",
+                               SDTIntToFPOp, [SDNPHasChain]>;
+def strict_fp_to_f16  : SDNode<"ISD::STRICT_FP_TO_FP16",
+                               SDTFPToIntOp, [SDNPHasChain]>;
+
 def strict_fsetcc  : SDNode<"ISD::STRICT_FSETCC",  SDTSetCC, [SDNPHasChain]>;
 def strict_fsetccs : SDNode<"ISD::STRICT_FSETCCS", SDTSetCC, [SDNPHasChain]>;
 
@@ -1576,6 +1582,13 @@ def any_fsetccs : PatFrags<(ops node:$lhs, node:$rhs, node:$pred),
                           [(strict_fsetccs node:$lhs, node:$rhs, node:$pred),
                            (setcc node:$lhs, node:$rhs, node:$pred)]>;
 
+def any_f16_to_fp : PatFrags<(ops node:$src),
+                              [(f16_to_fp node:$src),
+                               (strict_f16_to_fp node:$src)]>;
+def any_fp_to_f16 : PatFrags<(ops node:$src),
+                              [(fp_to_f16 node:$src),
+                               (strict_fp_to_f16 node:$src)]>;
+
 multiclass binary_atomic_op_ord {
   def NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val),
       (!cast<SDPatternOperator>(NAME) node:$ptr, node:$val)> {
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 53dc2be825f2..2fe4d5eeb742 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -482,8 +482,7 @@ inline constexpr CpuInfo CpuInfos[] = {
      (AArch64::ExtensionBitset(
          {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP16,
           AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS,
-          AArch64::AEK_PROFILE, AArch64::AEK_FLAGM, AArch64::AEK_PAUTH,
-          AArch64::AEK_FP16FML}))},
+          AArch64::AEK_PROFILE, AArch64::AEK_FLAGM, AArch64::AEK_PAUTH}))},
     {"cortex-a710", ARMV9A,
      (AArch64::ExtensionBitset(
          {AArch64::AEK_MTE, AArch64::AEK_PAUTH, AArch64::AEK_FLAGM,
@@ -514,13 +513,13 @@ inline constexpr CpuInfo CpuInfos[] = {
      (AArch64::ExtensionBitset(
          {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP16,
           AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS,
-          AArch64::AEK_PAUTH, AArch64::AEK_PROFILE}))},
+          AArch64::AEK_PAUTH, AArch64::AEK_PROFILE, AArch64::AEK_FLAGM}))},
     {"cortex-x2", ARMV9A,
      (AArch64::ExtensionBitset(
          {AArch64::AEK_MTE, AArch64::AEK_BF16, AArch64::AEK_I8MM,
           AArch64::AEK_PAUTH, AArch64::AEK_SSBS, AArch64::AEK_SB,
           AArch64::AEK_SVE, AArch64::AEK_SVE2, AArch64::AEK_SVE2BITPERM,
-          AArch64::AEK_FP16FML}))},
+          AArch64::AEK_FP16FML, AArch64::AEK_FLAGM}))},
     {"cortex-x3", ARMV9A,
      (AArch64::ExtensionBitset(
          {AArch64::AEK_SVE, AArch64::AEK_PERFMON, AArch64::AEK_PROFILE,
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 47904621c096..95014a546f72 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -434,6 +434,12 @@ public:
   /// string (separated by a '-' if the environment component is present).
   StringRef getOSAndEnvironmentName() const;
 
+  /// Get the version component of the environment component as a single
+  /// string (the version after the environment).
+  ///
+  /// For example, "fooos1.2.3" would return "1.2.3".
+  StringRef getEnvironmentVersionString() const;
+
   /// @}
   /// @name Convenience Predicates
   /// @{
diff --git a/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h b/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h
index 4136c45e1905..6bc01ececcf3 100644
--- a/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h
+++ b/llvm/include/llvm/Transforms/Scalar/IVUsersPrinter.h
@@ -25,6 +25,7 @@ public:
   explicit IVUsersPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &U);
+  static bool isRequired() { return true; }
 };
 }
 
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h b/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
index 4d1f934ae91d..f445e0696b5f 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
@@ -24,6 +24,7 @@ class LoopAccessInfoPrinterPass
 public:
   explicit LoopAccessInfoPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
index b433d2ec89dc..365b215c051f 100644
--- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -215,11 +215,13 @@ class PredicateInfoPrinterPass
 public:
   explicit PredicateInfoPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 /// Verifier pass for \c PredicateInfo.
 struct PredicateInfoVerifierPass : PassInfoMixin<PredicateInfoVerifierPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
 };
 
 } // end namespace llvm