vendor/llvm/llvm-trunk-r307894

99 files changed, 2313 insertions, 698 deletions

diff --git a/include/llvm-c/OrcBindings.h b/include/llvm-c/OrcBindings.h
index d86ea88088893..7ee3954313584 100644
--- a/include/llvm-c/OrcBindings.h
+++ b/include/llvm-c/OrcBindings.h
@@ -113,8 +113,9 @@ void LLVMOrcDisposeMangledSymbol(char *MangledSymbol);
 /**
  * Create a lazy compile callback.
  */
-LLVMOrcTargetAddress
+LLVMOrcErrorCode
 LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack,
+                                 LLVMOrcTargetAddress *RetAddr,
                                  LLVMOrcLazyCompileCallbackFn Callback,
                                  void *CallbackCtx);
 
@@ -135,8 +136,9 @@ LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack,
 /**
  * Add module to be eagerly compiled.
  */
-LLVMOrcModuleHandle
+LLVMOrcErrorCode
 LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack,
+                            LLVMOrcModuleHandle *RetHandle,
                             LLVMSharedModuleRef Mod,
                             LLVMOrcSymbolResolverFn SymbolResolver,
                             void *SymbolResolverCtx);
@@ -144,8 +146,9 @@ LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack,
 /**
  * Add module to be lazily compiled one function at a time.
  */
-LLVMOrcModuleHandle
+LLVMOrcErrorCode
 LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack,
+                           LLVMOrcModuleHandle *RetHandle,
                            LLVMSharedModuleRef Mod,
                            LLVMOrcSymbolResolverFn SymbolResolver,
                            void *SymbolResolverCtx);
@@ -153,10 +156,11 @@ LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack,
 /**
  * Add an object file.
  */
-LLVMOrcModuleHandle LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack,
-                                         LLVMSharedObjectBufferRef Obj,
-                                         LLVMOrcSymbolResolverFn SymbolResolver,
-                                         void *SymbolResolverCtx);
+LLVMOrcErrorCode LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack,
+                                      LLVMOrcModuleHandle *RetHandle,
+                                      LLVMSharedObjectBufferRef Obj,
+                                      LLVMOrcSymbolResolverFn SymbolResolver,
+                                      void *SymbolResolverCtx);
 
 /**
  * Remove a module set from the JIT.
@@ -164,18 +168,20 @@ LLVMOrcModuleHandle LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack,
  * This works for all modules that can be added via OrcAdd*, including object
  * files.
  */
-void LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, LLVMOrcModuleHandle H);
+LLVMOrcErrorCode LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack,
+                                     LLVMOrcModuleHandle H);
 
 /**
  * Get symbol address from JIT instance.
  */
-LLVMOrcTargetAddress LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack,
-                                             const char *SymbolName);
+LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack,
+                                         LLVMOrcTargetAddress *RetAddr,
+                                         const char *SymbolName);
 
 /**
  * Dispose of an ORC JIT stack.
  */
-void LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack);
+LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack);
 
 #ifdef __cplusplus
 }
diff --git a/include/llvm/ADT/APInt.h b/include/llvm/ADT/APInt.h
index e5f0c35534ac9..a1cce6e5fe170 100644
--- a/include/llvm/ADT/APInt.h
+++ b/include/llvm/ADT/APInt.h
@@ -401,7 +401,11 @@ public:
   /// \brief Determine if this is a value of 1.
   ///
   /// This checks to see if the value of this APInt is one.
-  bool isOneValue() const { return getActiveBits() == 1; }
+  bool isOneValue() const {
+    if (isSingleWord())
+      return U.VAL == 1;
+    return countLeadingZerosSlowCase() == BitWidth - 1;
+  }
 
   /// \brief Determine if this is the largest unsigned value.
   ///
diff --git a/include/llvm/ADT/STLExtras.h b/include/llvm/ADT/STLExtras.h
index 8c28412bb6078..83f289c42a23a 100644
--- a/include/llvm/ADT/STLExtras.h
+++ b/include/llvm/ADT/STLExtras.h
@@ -100,6 +100,8 @@ class function_ref<Ret(Params...)> {
   }
 
 public:
+  function_ref() : callback(nullptr) {}
+
   template <typename Callable>
   function_ref(Callable &&callable,
                typename std::enable_if<
@@ -110,6 +112,8 @@ public:
   Ret operator()(Params ...params) const {
     return callback(callable, std::forward<Params>(params)...);
   }
+
+  operator bool() const { return callback; }
 };
 
 // deleter - Very very very simple method that is used to invoke operator
diff --git a/include/llvm/ADT/SmallPtrSet.h b/include/llvm/ADT/SmallPtrSet.h
index a2ad74b1e04ae..4e8a2490ee3c5 100644
--- a/include/llvm/ADT/SmallPtrSet.h
+++ b/include/llvm/ADT/SmallPtrSet.h
@@ -15,9 +15,9 @@
 #ifndef LLVM_ADT_SMALLPTRSET_H
 #define LLVM_ADT_SMALLPTRSET_H
 
-#include "llvm/Config/abi-breaking.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
+#include "llvm/Support/ReverseIteration.h"
 #include "llvm/Support/type_traits.h"
 #include <cassert>
 #include <cstddef>
@@ -29,15 +29,6 @@
 
 namespace llvm {
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
-template <class T = void> struct ReverseIterate { static bool value; };
-#if LLVM_ENABLE_REVERSE_ITERATION
-template <class T> bool ReverseIterate<T>::value = true;
-#else
-template <class T> bool ReverseIterate<T>::value = false;
-#endif
-#endif
-
 /// SmallPtrSetImplBase - This is the common code shared among all the
 /// SmallPtrSet<>'s, which is almost everything.  SmallPtrSet has two modes, one
 /// for small and one for large sets.
diff --git a/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 3e05e09900a5f..5de3821242e0f 100644
--- a/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1353,4 +1353,4 @@ struct BFIDOTGraphTraitsBase : public DefaultDOTGraphTraits {
 
 #undef DEBUG_TYPE
 
-#endif
+#endif // LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H
diff --git a/include/llvm/Analysis/CGSCCPassManager.h b/include/llvm/Analysis/CGSCCPassManager.h
index a15a9e18c8153..32868cbecdcf0 100644
--- a/include/llvm/Analysis/CGSCCPassManager.h
+++ b/include/llvm/Analysis/CGSCCPassManager.h
@@ -577,12 +577,17 @@ public:
       // analyses will eventually occur when the module pass completes.
       PA.intersect(std::move(PassPA));
 
-      // Update the call graph based on this function pass. This may also
-      // update the current SCC to point to a smaller, more refined SCC.
-      CurrentC = &updateCGAndAnalysisManagerForFunctionPass(
-          CG, *CurrentC, *N, AM, UR, DebugLogging);
-      assert(CG.lookupSCC(*N) == CurrentC &&
-             "Current SCC not updated to the SCC containing the current node!");
+      // If the call graph hasn't been preserved, update it based on this
+      // function pass. This may also update the current SCC to point to
+      // a smaller, more refined SCC.
+      auto PAC = PA.getChecker<LazyCallGraphAnalysis>();
+      if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Module>>()) {
+        CurrentC = &updateCGAndAnalysisManagerForFunctionPass(
+            CG, *CurrentC, *N, AM, UR, DebugLogging);
+        assert(
+            CG.lookupSCC(*N) == CurrentC &&
+            "Current SCC not updated to the SCC containing the current node!");
+      }
     }
 
     // By definition we preserve the proxy. And we preserve all analyses on
diff --git a/include/llvm/Analysis/InlineCost.h b/include/llvm/Analysis/InlineCost.h
index ce0b7895f253c..f33a2de5a5f4a 100644
--- a/include/llvm/Analysis/InlineCost.h
+++ b/include/llvm/Analysis/InlineCost.h
@@ -160,7 +160,7 @@ InlineParams getInlineParams(int Threshold);
 /// the -Oz flag.
 InlineParams getInlineParams(unsigned OptLevel, unsigned SizeOptLevel);
 
-/// Return the cost associated with a callsite, including paramater passing
+/// Return the cost associated with a callsite, including parameter passing
 /// and the call/return instruction.
 int getCallsiteCost(CallSite CS, const DataLayout &DL);
 
diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h
index ad7f5c80549fc..3a052761ad7d8 100644
--- a/include/llvm/Analysis/LazyCallGraph.h
+++ b/include/llvm/Analysis/LazyCallGraph.h
@@ -652,17 +652,23 @@ public:
     /// Make an existing internal ref edge into a call edge.
     ///
     /// This may form a larger cycle and thus collapse SCCs into TargetN's SCC.
-    /// If that happens, the deleted SCC pointers are returned. These SCCs are
-    /// not in a valid state any longer but the pointers will remain valid
-    /// until destruction of the parent graph instance for the purpose of
-    /// clearing cached information.
+    /// If that happens, the optional callback \p MergedCB will be invoked (if
+    /// provided) on the SCCs being merged away prior to actually performing
+    /// the merge. Note that this will never include the target SCC as that
+    /// will be the SCC functions are merged into to resolve the cycle. Once
+    /// this function returns, these merged SCCs are not in a valid state but
+    /// the pointers will remain valid until destruction of the parent graph
+    /// instance for the purpose of clearing cached information. This function
+    /// also returns 'true' if a cycle was formed and some SCCs merged away as
+    /// a convenience.
     ///
     /// After this operation, both SourceN's SCC and TargetN's SCC may move
     /// position within this RefSCC's postorder list. Any SCCs merged are
     /// merged into the TargetN's SCC in order to preserve reachability analyses
     /// which took place on that SCC.
-    SmallVector<SCC *, 1> switchInternalEdgeToCall(Node &SourceN,
-                                                   Node &TargetN);
+    bool switchInternalEdgeToCall(
+        Node &SourceN, Node &TargetN,
+        function_ref<void(ArrayRef<SCC *> MergedSCCs)> MergeCB = {});
 
     /// Make an existing internal call edge between separate SCCs into a ref
     /// edge.
diff --git a/include/llvm/Analysis/MemoryBuiltins.h b/include/llvm/Analysis/MemoryBuiltins.h
index 60dafccd84bdf..23ab372703eea 100644
--- a/include/llvm/Analysis/MemoryBuiltins.h
+++ b/include/llvm/Analysis/MemoryBuiltins.h
@@ -224,6 +224,9 @@ public:
   SizeOffsetType visitSelectInst(SelectInst &I);
   SizeOffsetType visitUndefValue(UndefValue&);
   SizeOffsetType visitInstruction(Instruction &I);
+
+private:
+  bool CheckedZextOrTrunc(APInt &I);
 };
 
 typedef std::pair<Value*, Value*> SizeOffsetEvalType;
diff --git a/include/llvm/Analysis/RegionInfoImpl.h b/include/llvm/Analysis/RegionInfoImpl.h
index c0337b6daf37f..cd4ec0a03a9ee 100644
--- a/include/llvm/Analysis/RegionInfoImpl.h
+++ b/include/llvm/Analysis/RegionInfoImpl.h
@@ -34,10 +34,10 @@
 #include <type_traits>
 #include <vector>
 
-namespace llvm {
-
 #define DEBUG_TYPE "region"
 
+namespace llvm {
+
 //===----------------------------------------------------------------------===//
 /// RegionBase Implementation
 template <class Tr>
@@ -901,8 +901,8 @@ void RegionInfoBase<Tr>::calculate(FuncT &F) {
   buildRegionsTree(DT->getNode(BB), TopLevelRegion);
 }
 
-#undef DEBUG_TYPE
-
 } // end namespace llvm
 
+#undef DEBUG_TYPE
+
 #endif // LLVM_ANALYSIS_REGIONINFOIMPL_H
diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h
index 68fbf640994c2..dfb525e3de7ac 100644
--- a/include/llvm/Analysis/TargetTransformInfo.h
+++ b/include/llvm/Analysis/TargetTransformInfo.h
@@ -753,6 +753,28 @@ public:
   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
                                            Type *ExpectedType) const;
 
+  /// \returns The type to use in a loop expansion of a memcpy call.
+  Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
+                                  unsigned SrcAlign, unsigned DestAlign) const;
+
+  /// \param[out] OpsOut The operand types to copy RemainingBytes of memory.
+  /// \param RemainingBytes The number of bytes to copy.
+  ///
+  /// Calculates the operand types to use when copying \p RemainingBytes of
+  /// memory, where source and destination alignments are \p SrcAlign and
+  /// \p DestAlign respectively.
+  void getMemcpyLoopResidualLoweringType(SmallVectorImpl<Type *> &OpsOut,
+                                         LLVMContext &Context,
+                                         unsigned RemainingBytes,
+                                         unsigned SrcAlign,
+                                         unsigned DestAlign) const;
+
+  /// \returns True if we want to test the new memcpy lowering functionality in
+  /// Transform/Utils.
+  /// Temporary. Will be removed once we move to the new functionality and
+  /// remove the old.
+  bool useWideIRMemcpyLoopLowering() const;
+
   /// \returns True if the two functions have compatible attributes for inlining
   /// purposes.
   bool areInlineCompatible(const Function *Caller,
@@ -953,6 +975,12 @@ public:
   virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0;
   virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
                                                    Type *ExpectedType) = 0;
+  virtual Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
+                                          unsigned SrcAlign,
+                                          unsigned DestAlign) const = 0;
+  virtual void getMemcpyLoopResidualLoweringType(
+      SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
+      unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const = 0;
   virtual bool areInlineCompatible(const Function *Caller,
                                    const Function *Callee) const = 0;
   virtual unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const = 0;
@@ -1266,6 +1294,19 @@ public:
                                            Type *ExpectedType) override {
     return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
   }
+  Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
+                                  unsigned SrcAlign,
+                                  unsigned DestAlign) const override {
+    return Impl.getMemcpyLoopLoweringType(Context, Length, SrcAlign, DestAlign);
+  }
+  void getMemcpyLoopResidualLoweringType(SmallVectorImpl<Type *> &OpsOut,
+                                         LLVMContext &Context,
+                                         unsigned RemainingBytes,
+                                         unsigned SrcAlign,
+                                         unsigned DestAlign) const override {
+    Impl.getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
+                                           SrcAlign, DestAlign);
+  }
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const override {
     return Impl.areInlineCompatible(Caller, Callee);
diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h
index 0246fc1c02cce..8740ee92eed5f 100644
--- a/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -444,6 +444,20 @@ public:
     return nullptr;
   }
 
+  Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
+                                  unsigned SrcAlign, unsigned DestAlign) const {
+    return Type::getInt8Ty(Context);
+  }
+
+  void getMemcpyLoopResidualLoweringType(SmallVectorImpl<Type *> &OpsOut,
+                                         LLVMContext &Context,
+                                         unsigned RemainingBytes,
+                                         unsigned SrcAlign,
+                                         unsigned DestAlign) const {
+    for (unsigned i = 0; i != RemainingBytes; ++i)
+      OpsOut.push_back(Type::getInt8Ty(Context));
+  }
+
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const {
     return (Caller->getFnAttribute("target-cpu") ==
diff --git a/include/llvm/Analysis/ValueTracking.h b/include/llvm/Analysis/ValueTracking.h
index e953ec8ab6abe..f4c57d4289fc8 100644
--- a/include/llvm/Analysis/ValueTracking.h
+++ b/include/llvm/Analysis/ValueTracking.h
@@ -523,8 +523,7 @@ template <typename T> class ArrayRef;
   /// (A)
   Optional<bool> isImpliedCondition(const Value *LHS, const Value *RHS,
                                     const DataLayout &DL,
-                                    bool InvertAPred = false,
-                                    unsigned Depth = 0,
+                                    bool LHSIsFalse = false, unsigned Depth = 0,
                                     AssumptionCache *AC = nullptr,
                                     const Instruction *CxtI = nullptr,
                                     const DominatorTree *DT = nullptr);
diff --git a/include/llvm/BinaryFormat/Wasm.h b/include/llvm/BinaryFormat/Wasm.h
index eef473b20dde1..23e30b7a868d9 100644
--- a/include/llvm/BinaryFormat/Wasm.h
+++ b/include/llvm/BinaryFormat/Wasm.h
@@ -94,7 +94,7 @@ struct WasmFunction {
 };
 
 struct WasmDataSegment {
-  uint32_t Index;
+  uint32_t MemoryIndex;
   WasmInitExpr Offset;
   ArrayRef<uint8_t> Content;
 };
@@ -107,7 +107,7 @@ struct WasmElemSegment {
 
 struct WasmRelocation {
   uint32_t Type;   // The type of the relocation.
-  int32_t Index;   // Index into function to global index space.
+  uint32_t Index;  // Index into function to global index space.
   uint64_t Offset; // Offset from the start of the section.
   int64_t Addend;  // A value to add to the symbol.
 };
diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h
index 5435e48ff4242..3777f956cf279 100644
--- a/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/include/llvm/Bitcode/LLVMBitCodes.h
@@ -59,6 +59,8 @@ enum BlockIDs {
   FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
 
   SYMTAB_BLOCK_ID,
+
+  SYNC_SCOPE_NAMES_BLOCK_ID,
 };
 
 /// Identification block contains a string that describes the producer details,
@@ -172,6 +174,10 @@ enum OperandBundleTagCode {
   OPERAND_BUNDLE_TAG = 1, // TAG: [strchr x N]
 };
 
+enum SyncScopeNameCode {
+  SYNC_SCOPE_NAME = 1,
+};
+
 // Value symbol table codes.
 enum ValueSymtabCodes {
   VST_CODE_ENTRY = 1,   // VST_ENTRY: [valueid, namechar x N]
@@ -404,12 +410,6 @@ enum AtomicOrderingCodes {
   ORDERING_SEQCST = 6
 };
 
-/// Encoded SynchronizationScope values.
-enum AtomicSynchScopeCodes {
-  SYNCHSCOPE_SINGLETHREAD = 0,
-  SYNCHSCOPE_CROSSTHREAD = 1
-};
-
 /// Markers and flags for call instruction.
 enum CallMarkersFlags {
   CALL_TAIL = 0,
diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index c898667f14742..60bbc9aaa5bd4 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -608,8 +608,8 @@ private:
   // Internal Implementation Details
   //===------------------------------------------------------------------===//
 
-  /// This emits visibility information about symbol, if this is suported by the
-  /// target.
+  /// This emits visibility information about symbol, if this is supported by
+  /// the target.
   void EmitVisibility(MCSymbol *Sym, unsigned Visibility,
                       bool IsDefinition = true) const;
 
diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h
index a740df96899df..b59fd60e8aed3 100644
--- a/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/include/llvm/CodeGen/BasicTTIImpl.h
@@ -428,7 +428,7 @@ public:
 
     std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
 
-    bool IsFloat = Ty->getScalarType()->isFloatingPointTy();
+    bool IsFloat = Ty->isFPOrFPVectorTy();
     // Assume that floating point arithmetic operations cost twice as much as
     // integer operations.
     unsigned OpCost = (IsFloat ? 2 : 1);
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
index ec60123e54b15..59a4073646ebb 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
@@ -16,14 +16,17 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H
 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H
 
+#include "llvm/ADT/SmallVector.h"
 #include <bitset>
 #include <cstddef>
 #include <cstdint>
 #include <functional>
 #include <initializer_list>
+#include <vector>
 
 namespace llvm {
 
+class LLT;
 class MachineInstr;
 class MachineInstrBuilder;
 class MachineOperand;
@@ -58,6 +61,131 @@ public:
   }
 };
 
+enum {
+  /// Record the specified instruction
+  /// - NewInsnID - Instruction ID to define
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  GIM_RecordInsn,
+
+  /// Check the feature bits
+  /// - Expected features
+  GIM_CheckFeatures,
+
+  /// Check the opcode on the specified instruction
+  /// - InsnID - Instruction ID
+  /// - Expected opcode
+  GIM_CheckOpcode,
+  /// Check the instruction has the right number of operands
+  /// - InsnID - Instruction ID
+  /// - Expected number of operands
+  GIM_CheckNumOperands,
+
+  /// Check the type for the specified operand
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - Expected type
+  GIM_CheckType,
+  /// Check the register bank for the specified operand
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - Expected register bank (specified as a register class)
+  GIM_CheckRegBankForClass,
+  /// Check the operand matches a complex predicate
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - RendererID - The renderer to hold the result
+  /// - Complex predicate ID
+  GIM_CheckComplexPattern,
+  /// Check the operand is a specific integer
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - Expected integer
+  GIM_CheckConstantInt,
+  /// Check the operand is a specific literal integer (i.e. MO.isImm() or MO.isCImm() is true).
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - Expected integer
+  GIM_CheckLiteralInt,
+  /// Check the operand is a specific intrinsic ID
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - Expected Intrinsic ID
+  GIM_CheckIntrinsicID,
+  /// Check the specified operand is an MBB
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  GIM_CheckIsMBB,
+
+  /// Check if the specified operand is safe to fold into the current
+  /// instruction.
+  /// - InsnID - Instruction ID
+  GIM_CheckIsSafeToFold,
+
+  //=== Renderers ===
+
+  /// Mutate an instruction
+  /// - NewInsnID - Instruction ID to define
+  /// - OldInsnID - Instruction ID to mutate
+  /// - NewOpcode - The new opcode to use
+  GIR_MutateOpcode,
+  /// Build a new instruction
+  /// - InsnID - Instruction ID to define
+  /// - Opcode - The new opcode to use
+  GIR_BuildMI,
+
+  /// Copy an operand to the specified instruction
+  /// - NewInsnID - Instruction ID to modify
+  /// - OldInsnID - Instruction ID to copy from
+  /// - OpIdx - The operand to copy
+  GIR_Copy,
+  /// Copy an operand to the specified instruction
+  /// - NewInsnID - Instruction ID to modify
+  /// - OldInsnID - Instruction ID to copy from
+  /// - OpIdx - The operand to copy
+  /// - SubRegIdx - The subregister to copy
+  GIR_CopySubReg,
+  /// Add an implicit register def to the specified instruction
+  /// - InsnID - Instruction ID to modify
+  /// - RegNum - The register to add
+  GIR_AddImplicitDef,
+  /// Add an implicit register use to the specified instruction
+  /// - InsnID - Instruction ID to modify
+  /// - RegNum - The register to add
+  GIR_AddImplicitUse,
+  /// Add an register to the specified instruction
+  /// - InsnID - Instruction ID to modify
+  /// - RegNum - The register to add
+  GIR_AddRegister,
+  /// Add an immediate to the specified instruction
+  /// - InsnID - Instruction ID to modify
+  /// - Imm - The immediate to add
+  GIR_AddImm,
+  /// Render complex operands to the specified instruction
+  /// - InsnID - Instruction ID to modify
+  /// - RendererID - The renderer to call
+  GIR_ComplexRenderer,
+
+  /// Constrain an instruction operand to a register class.
+  /// - InsnID - Instruction ID to modify
+  /// - OpIdx - Operand index
+  /// - RCEnum - Register class enumeration value
+  GIR_ConstrainOperandRC,
+  /// Constrain an instructions operands according to the instruction
+  /// description.
+  /// - InsnID - Instruction ID to modify
+  GIR_ConstrainSelectedInstOperands,
+  /// Merge all memory operands into instruction.
+  /// - InsnID - Instruction ID to modify
+  GIR_MergeMemOperands,
+  /// Erase from parent.
+  /// - InsnID - Instruction ID to erase
+  GIR_EraseFromParent,
+
+  /// A successful emission
+  GIR_Done,
+};
+
 /// Provides the logic to select generic machine instructions.
 class InstructionSelector {
 public:
@@ -78,9 +206,39 @@ public:
 
 protected:
   using ComplexRendererFn = std::function<void(MachineInstrBuilder &)>;
+  using RecordedMIVector = SmallVector<MachineInstr *, 4>;
+  using NewMIVector = SmallVector<MachineInstrBuilder, 4>;
+
+  struct MatcherState {
+    std::vector<ComplexRendererFn> Renderers;
+    RecordedMIVector MIs;
+
+    MatcherState(unsigned MaxRenderers);
+  };
 
+public:
+  template <class PredicateBitset, class ComplexMatcherMemFn>
+  struct MatcherInfoTy {
+    const LLT *TypeObjects;
+    const PredicateBitset *FeatureBitsets;
+    const std::vector<ComplexMatcherMemFn> ComplexPredicates;
+  };
+
+protected:
   InstructionSelector();
 
+  /// Execute a given matcher table and return true if the match was successful
+  /// and false otherwise.
+  template <class TgtInstructionSelector, class PredicateBitset,
+            class ComplexMatcherMemFn>
+  bool executeMatchTable(
+      TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
+      const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo,
+      const int64_t *MatchTable, const TargetInstrInfo &TII,
+      MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
+      const RegisterBankInfo &RBI,
+      const PredicateBitset &AvailableFeatures) const;
+
   /// Constrain a register operand of an instruction \p I to a specified
   /// register class. This could involve inserting COPYs before (for uses) or
   /// after (for defs) and may replace the operand of \p I.
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
new file mode 100644
index 0000000000000..98b6b859b9e26
--- /dev/null
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -0,0 +1,337 @@
+//==-- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h ---------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file This file declares the API for the instruction selector.
+/// This class is responsible for selecting machine instructions.
+/// It's implemented by the target. It's used by the InstructionSelect pass.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
+#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
+
+namespace llvm {
+template <class TgtInstructionSelector, class PredicateBitset,
+          class ComplexMatcherMemFn>
+bool InstructionSelector::executeMatchTable(
+    TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
+    const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo,
+    const int64_t *MatchTable, const TargetInstrInfo &TII,
+    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
+    const RegisterBankInfo &RBI,
+    const PredicateBitset &AvailableFeatures) const {
+  const int64_t *Command = MatchTable;
+  while (true) {
+    switch (*Command++) {
+    case GIM_RecordInsn: {
+      int64_t NewInsnID = *Command++;
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+
+      // As an optimisation we require that MIs[0] is always the root. Refuse
+      // any attempt to modify it.
+      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
+      (void)NewInsnID;
+
+      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
+      if (!MO.isReg()) {
+        DEBUG(dbgs() << "Rejected (not a register)\n");
+        return false;
+      }
+      if (TRI.isPhysicalRegister(MO.getReg())) {
+        DEBUG(dbgs() << "Rejected (is a physical register)\n");
+        return false;
+      }
+
+      assert((size_t)NewInsnID == State.MIs.size() &&
+             "Expected to store MIs in order");
+      State.MIs.push_back(MRI.getVRegDef(MO.getReg()));
+      DEBUG(dbgs() << "MIs[" << NewInsnID << "] = GIM_RecordInsn(" << InsnID
+                   << ", " << OpIdx << ")\n");
+      break;
+    }
+
+    case GIM_CheckFeatures: {
+      int64_t ExpectedBitsetID = *Command++;
+      DEBUG(dbgs() << "GIM_CheckFeatures(ExpectedBitsetID=" << ExpectedBitsetID
+                   << ")\n");
+      if ((AvailableFeatures & MatcherInfo.FeatureBitsets[ExpectedBitsetID]) !=
+          MatcherInfo.FeatureBitsets[ExpectedBitsetID]) {
+        DEBUG(dbgs() << "Rejected\n");
+        return false;
+      }
+      break;
+    }
+
+    case GIM_CheckOpcode: {
+      int64_t InsnID = *Command++;
+      int64_t Expected = *Command++;
+
+      unsigned Opcode = State.MIs[InsnID]->getOpcode();
+      DEBUG(dbgs() << "GIM_CheckOpcode(MIs[" << InsnID << "], ExpectedOpcode="
+                   << Expected << ") // Got=" << Opcode << "\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (Opcode != Expected)
+        return false;
+      break;
+    }
+    case GIM_CheckNumOperands: {
+      int64_t InsnID = *Command++;
+      int64_t Expected = *Command++;
+      DEBUG(dbgs() << "GIM_CheckNumOperands(MIs[" << InsnID
+                   << "], Expected=" << Expected << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (State.MIs[InsnID]->getNumOperands() != Expected)
+        return false;
+      break;
+    }
+
+    case GIM_CheckType: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t TypeID = *Command++;
+      DEBUG(dbgs() << "GIM_CheckType(MIs[" << InsnID << "]->getOperand("
+                   << OpIdx << "), TypeID=" << TypeID << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) !=
+          MatcherInfo.TypeObjects[TypeID])
+        return false;
+      break;
+    }
+    case GIM_CheckRegBankForClass: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t RCEnum = *Command++;
+      DEBUG(dbgs() << "GIM_CheckRegBankForClass(MIs[" << InsnID
+                   << "]->getOperand(" << OpIdx << "), RCEnum=" << RCEnum
+                   << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (&RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
+          RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, TRI))
+        return false;
+      break;
+    }
+    case GIM_CheckComplexPattern: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t RendererID = *Command++;
+      int64_t ComplexPredicateID = *Command++;
+      DEBUG(dbgs() << "State.Renderers[" << RendererID
+                   << "] = GIM_CheckComplexPattern(MIs[" << InsnID
+                   << "]->getOperand(" << OpIdx
+                   << "), ComplexPredicateID=" << ComplexPredicateID << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      // FIXME: Use std::invoke() when it's available.
+      if (!(State.Renderers[RendererID] =
+                (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])(
+                    State.MIs[InsnID]->getOperand(OpIdx))))
+        return false;
+      break;
+    }
+    case GIM_CheckConstantInt: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t Value = *Command++;
+      DEBUG(dbgs() << "GIM_CheckConstantInt(MIs[" << InsnID << "]->getOperand("
+                   << OpIdx << "), Value=" << Value << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, MRI))
+        return false;
+      break;
+    }
+    case GIM_CheckLiteralInt: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t Value = *Command++;
+      DEBUG(dbgs() << "GIM_CheckLiteralInt(MIs[" << InsnID << "]->getOperand(" << OpIdx
+                   << "), Value=" << Value << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx);
+      if (!OM.isCImm() || !OM.getCImm()->equalsInt(Value))
+        return false;
+      break;
+    }
+    case GIM_CheckIntrinsicID: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t Value = *Command++;
+      DEBUG(dbgs() << "GIM_CheckIntrinsicID(MIs[" << InsnID << "]->getOperand(" << OpIdx
+                   << "), Value=" << Value << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx);
+      if (!OM.isIntrinsicID() || OM.getIntrinsicID() != Value)
+        return false;
+      break;
+    }
+    case GIM_CheckIsMBB: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      DEBUG(dbgs() << "GIM_CheckIsMBB(MIs[" << InsnID << "]->getOperand("
+                   << OpIdx << "))\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB())
+        return false;
+      break;
+    }
+
+    case GIM_CheckIsSafeToFold: {
+      int64_t InsnID = *Command++;
+      DEBUG(dbgs() << "GIM_CheckIsSafeToFold(MIs[" << InsnID << "])\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (!isObviouslySafeToFold(*State.MIs[InsnID]))
+        return false;
+      break;
+    }
+
+    case GIR_MutateOpcode: {
+      int64_t OldInsnID = *Command++;
+      int64_t NewInsnID = *Command++;
+      int64_t NewOpcode = *Command++;
+      assert((size_t)NewInsnID == OutMIs.size() &&
+             "Expected to store MIs in order");
+      OutMIs.push_back(
+          MachineInstrBuilder(*State.MIs[OldInsnID]->getParent()->getParent(),
+                              State.MIs[OldInsnID]));
+      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
+      DEBUG(dbgs() << "GIR_MutateOpcode(OutMIs[" << NewInsnID << "], MIs["
+                   << OldInsnID << "], " << NewOpcode << ")\n");
+      break;
+    }
+    case GIR_BuildMI: {
+      int64_t InsnID = *Command++;
+      int64_t Opcode = *Command++;
+      assert((size_t)InsnID == OutMIs.size() &&
+             "Expected to store MIs in order");
+      (void)InsnID;
+      OutMIs.push_back(BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
+                               State.MIs[0]->getDebugLoc(), TII.get(Opcode)));
+      DEBUG(dbgs() << "GIR_BuildMI(OutMIs[" << InsnID << "], " << Opcode
+                   << ")\n");
+      break;
+    }
+
+    case GIR_Copy: {
+      int64_t NewInsnID = *Command++;
+      int64_t OldInsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
+      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
+      DEBUG(dbgs() << "GIR_Copy(OutMIs[" << NewInsnID << "], MIs[" << OldInsnID
+                   << "], " << OpIdx << ")\n");
+      break;
+    }
+    case GIR_CopySubReg: {
+      int64_t NewInsnID = *Command++;
+      int64_t OldInsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t SubRegIdx = *Command++;
+      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
+      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
+                               0, SubRegIdx);
+      DEBUG(dbgs() << "GIR_CopySubReg(OutMIs[" << NewInsnID << "], MIs["
+                   << OldInsnID << "], " << OpIdx << ", " << SubRegIdx
+                   << ")\n");
+      break;
+    }
+    case GIR_AddImplicitDef: {
+      int64_t InsnID = *Command++;
+      int64_t RegNum = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
+      DEBUG(dbgs() << "GIR_AddImplicitDef(OutMIs[" << InsnID << "], " << RegNum
+                   << ")\n");
+      break;
+    }
+    case GIR_AddImplicitUse: {
+      int64_t InsnID = *Command++;
+      int64_t RegNum = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
+      DEBUG(dbgs() << "GIR_AddImplicitUse(OutMIs[" << InsnID << "], " << RegNum
+                   << ")\n");
+      break;
+    }
+    case GIR_AddRegister: {
+      int64_t InsnID = *Command++;
+      int64_t RegNum = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      OutMIs[InsnID].addReg(RegNum);
+      DEBUG(dbgs() << "GIR_AddRegister(OutMIs[" << InsnID << "], " << RegNum
+                   << ")\n");
+      break;
+    }
+    case GIR_AddImm: {
+      int64_t InsnID = *Command++;
+      int64_t Imm = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      OutMIs[InsnID].addImm(Imm);
+      DEBUG(dbgs() << "GIR_AddImm(OutMIs[" << InsnID << "], " << Imm << ")\n");
+      break;
+    }
+    case GIR_ComplexRenderer: {
+      int64_t InsnID = *Command++;
+      int64_t RendererID = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      State.Renderers[RendererID](OutMIs[InsnID]);
+      DEBUG(dbgs() << "GIR_ComplexRenderer(OutMIs[" << InsnID << "], "
+                   << RendererID << ")\n");
+      break;
+    }
+
+    case GIR_ConstrainOperandRC: {
+      int64_t InsnID = *Command++;
+      int64_t OpIdx = *Command++;
+      int64_t RCEnum = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
+                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
+      DEBUG(dbgs() << "GIR_ConstrainOperandRC(OutMIs[" << InsnID << "], "
+                   << OpIdx << ", " << RCEnum << ")\n");
+      break;
+    }
+    case GIR_ConstrainSelectedInstOperands: {
+      int64_t InsnID = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
+                                       RBI);
+      DEBUG(dbgs() << "GIR_ConstrainSelectedInstOperands(OutMIs[" << InsnID
+                   << "])\n");
+      break;
+    }
+    case GIR_MergeMemOperands: {
+      int64_t InsnID = *Command++;
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      for (const auto *FromMI : State.MIs)
+        for (const auto &MMO : FromMI->memoperands())
+          OutMIs[InsnID].addMemOperand(MMO);
+      DEBUG(dbgs() << "GIR_MergeMemOperands(OutMIs[" << InsnID << "])\n");
+      break;
+    }
+    case GIR_EraseFromParent: {
+      int64_t InsnID = *Command++;
+      assert(State.MIs[InsnID] &&
+             "Attempted to erase an undefined instruction");
+      State.MIs[InsnID]->eraseFromParent();
+      DEBUG(dbgs() << "GIR_EraseFromParent(MIs[" << InsnID << "])\n");
+      break;
+    }
+
+    case GIR_Done:
+      DEBUG(dbgs() << "GIR_Done");
+      return true;
+
+    default:
+      llvm_unreachable("Unexpected command");
+    }
+  }
+}
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 5197ba869c0a6..1fd45b52e3ac7 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -101,11 +101,11 @@ private:
   const LegalizerInfo &LI;
 };
 
-/// Helper function that replaces \p MI with a libcall.
+/// Helper function that creates the given libcall.
 LegalizerHelper::LegalizeResult
-replaceWithLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
-                   RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result,
-                   ArrayRef<CallLowering::ArgInfo> Args);
+createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
+              const CallLowering::ArgInfo &Result,
+              ArrayRef<CallLowering::ArgInfo> Args);
 
 } // End namespace llvm.
 
diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index c9327d50432e1..85e6fef1f3c26 100644
--- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugLoc.h"
 
@@ -59,6 +60,21 @@ class MachineIRBuilder {
   }
 
   void validateTruncExt(unsigned Dst, unsigned Src, bool IsExtend);
+  MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Res, unsigned Op0, unsigned Op1);
+
+  unsigned getDestFromArg(unsigned Reg) { return Reg; }
+  unsigned getDestFromArg(LLT Ty) {
+    return getMF().getRegInfo().createGenericVirtualRegister(Ty);
+  }
+  unsigned getDestFromArg(const TargetRegisterClass *RC) {
+    return getMF().getRegInfo().createVirtualRegister(RC);
+  }
+
+  unsigned getRegFromArg(unsigned Reg) { return Reg; }
+
+  unsigned getRegFromArg(const MachineInstrBuilder &MIB) {
+    return MIB->getOperand(0).getReg();
+  }
 
 public:
   /// Getter for the function we currently build.
@@ -120,6 +136,22 @@ public:
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildInstr(unsigned Opcode);
 
+  /// DAG like Generic method for building arbitrary instructions as above.
+  /// \Opc opcode for the instruction.
+  /// \Ty Either LLT/TargetRegisterClass/unsigned types for Dst
+  /// \Args Variadic list of uses of types(unsigned/MachineInstrBuilder)
+  /// Uses of type MachineInstrBuilder will perform
+  /// getOperand(0).getReg() to convert to register.
+  template <typename DstTy, typename... UseArgsTy>
+  MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty,
+                                 UseArgsTy &&... Args) {
+    auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty));
+    unsigned It[] = {(getRegFromArg(Args))...};
+    for (const auto &i : It)
+      MIB.addUse(i);
+    return MIB;
+  }
+
   /// Build but don't insert <empty> = \p Opcode <empty>.
   ///
   /// \pre setMF, setBasicBlock or setMI  must have been called.
@@ -188,6 +220,11 @@ public:
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildAdd(unsigned Res, unsigned Op0,
                                unsigned Op1);
+  template <typename DstTy, typename... UseArgsTy>
+  MachineInstrBuilder buildAdd(DstTy &&Ty, UseArgsTy &&... UseArgs) {
+    unsigned Res = getDestFromArg(Ty);
+    return buildAdd(Res, (getRegFromArg(UseArgs))...);
+  }
 
   /// Build and insert \p Res<def> = G_SUB \p Op0, \p Op1
   ///
@@ -295,6 +332,18 @@ public:
   MachineInstrBuilder buildAnd(unsigned Res, unsigned Op0,
                                unsigned Op1);
 
+  /// Build and insert \p Res<def> = G_OR \p Op0, \p Op1
+  ///
+  /// G_OR sets \p Res to the bitwise or of integer parameters \p Op0 and \p
+  /// Op1.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers
+  ///      with the same (scalar or vector) type).
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildOr(unsigned Res, unsigned Op0, unsigned Op1);
+
   /// Build and insert \p Res<def> = G_ANYEXT \p Op0
   ///
   /// G_ANYEXT produces a register of the specified width, with bits 0 to
@@ -416,6 +465,10 @@ public:
   /// \return The newly created instruction.
   MachineInstrBuilder buildConstant(unsigned Res, int64_t Val);
 
+  template <typename DstType>
+  MachineInstrBuilder buildConstant(DstType &&Res, int64_t Val) {
+    return buildConstant(getDestFromArg(Res), Val);
+  }
   /// Build and insert \p Res = G_FCONSTANT \p Val
   ///
   /// G_FCONSTANT is a floating-point constant with the specified size and
diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h
index fa1ec867ea3dc..c28b1a06854fc 100644
--- a/include/llvm/CodeGen/LiveRegUnits.h
+++ b/include/llvm/CodeGen/LiveRegUnits.h
@@ -93,12 +93,14 @@ public:
   }
 
   /// Updates liveness when stepping backwards over the instruction \p MI.
+  /// This removes all register units defined or clobbered in \p MI and then
+  /// adds the units used (as in use operands) in \p MI.
   void stepBackward(const MachineInstr &MI);
 
-  /// Mark all register units live during instruction \p MI.
-  /// This can be used to accumulate live/unoccupied registers over a range of
-  /// instructions.
-  void accumulateBackward(const MachineInstr &MI);
+  /// Adds all register units used, defined or clobbered in \p MI.
+  /// This is useful when walking over a range of instruction to find registers
+  /// unused over the whole range.
+  void accumulate(const MachineInstr &MI);
 
   /// Adds registers living out of block \p MBB.
   /// Live out registers are the union of the live-in registers of the successor
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index f67da7b01c54b..19173fa39bdc1 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -650,7 +650,7 @@ public:
       MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
       unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
       const MDNode *Ranges = nullptr,
-      SynchronizationScope SynchScope = CrossThread,
+      SyncScope::ID SSID = SyncScope::System,
       AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
       AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
 
diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h
index 78adce507b8cd..a9de0db05d72c 100644
--- a/include/llvm/CodeGen/MachineMemOperand.h
+++ b/include/llvm/CodeGen/MachineMemOperand.h
@@ -114,6 +114,9 @@ public:
     MOInvariant = 1u << 5,
 
     // Reserved for use by target-specific passes.
+    // Targets may override getSerializableMachineMemOperandTargetFlags() to
+    // enable MIR serialization/parsing of these flags.  If more of these flags
+    // are added, the MIR printing/parsing code will need to be updated as well.
     MOTargetFlag1 = 1u << 6,
     MOTargetFlag2 = 1u << 7,
     MOTargetFlag3 = 1u << 8,
@@ -124,8 +127,8 @@ public:
 private:
   /// Atomic information for this memory operation.
   struct MachineAtomicInfo {
-    /// Synchronization scope for this memory operation.
-    unsigned SynchScope : 1;      // enum SynchronizationScope
+    /// Synchronization scope ID for this memory operation.
+    unsigned SSID : 8;            // SyncScope::ID
     /// Atomic ordering requirements for this memory operation. For cmpxchg
     /// atomic operations, atomic ordering requirements when store occurs.
     unsigned Ordering : 4;        // enum AtomicOrdering
@@ -152,7 +155,7 @@ public:
                     unsigned base_alignment,
                     const AAMDNodes &AAInfo = AAMDNodes(),
                     const MDNode *Ranges = nullptr,
-                    SynchronizationScope SynchScope = CrossThread,
+                    SyncScope::ID SSID = SyncScope::System,
                     AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
                     AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
 
@@ -202,9 +205,9 @@ public:
   /// Return the range tag for the memory reference.
   const MDNode *getRanges() const { return Ranges; }
 
-  /// Return the synchronization scope for this memory operation.
-  SynchronizationScope getSynchScope() const {
-    return static_cast<SynchronizationScope>(AtomicInfo.SynchScope);
+  /// Returns the synchronization scope ID for this memory operation.
+  SyncScope::ID getSyncScopeID() const {
+    return static_cast<SyncScope::ID>(AtomicInfo.SSID);
   }
 
   /// Return the atomic ordering requirements for this memory operation. For
diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h
index 8c3aacaa8efc1..08151be110832 100644
--- a/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -340,6 +340,18 @@ namespace RTLIB {
     MEMCPY_ELEMENT_UNORDERED_ATOMIC_8,
     MEMCPY_ELEMENT_UNORDERED_ATOMIC_16,
 
+    MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1,
+    MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2,
+    MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4,
+    MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8,
+    MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16,
+    
+    MEMSET_ELEMENT_UNORDERED_ATOMIC_1,
+    MEMSET_ELEMENT_UNORDERED_ATOMIC_2,
+    MEMSET_ELEMENT_UNORDERED_ATOMIC_4,
+    MEMSET_ELEMENT_UNORDERED_ATOMIC_8,
+    MEMSET_ELEMENT_UNORDERED_ATOMIC_16,
+
     // EXCEPTION HANDLING
     UNWIND_RESUME,
 
@@ -515,6 +527,17 @@ namespace RTLIB {
   /// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
   /// UNKNOW_LIBCALL if there is none.
   Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+  /// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return
+  /// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+  /// UNKNOW_LIBCALL if there is none.
+  Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+  /// getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return
+  /// MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+  /// UNKNOW_LIBCALL if there is none.
+  Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
 }
 }
 
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index 4d72eda5c71ae..25afc5b506df6 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -235,6 +235,9 @@ class TargetRegisterInfo;
              "SDep::Output edge cannot use the zero register!");
       Contents.Reg = Reg;
     }
+
+    raw_ostream &print(raw_ostream &O,
+                       const TargetRegisterInfo *TRI = nullptr) const;
   };
 
   template <>
@@ -458,7 +461,10 @@ class TargetRegisterInfo;
 
     void dump(const ScheduleDAG *G) const;
     void dumpAll(const ScheduleDAG *G) const;
-    void print(raw_ostream &O, const ScheduleDAG *G) const;
+    raw_ostream &print(raw_ostream &O,
+                       const SUnit *N = nullptr,
+                       const SUnit *X = nullptr) const;
+    raw_ostream &print(raw_ostream &O, const ScheduleDAG *G) const;
 
   private:
     void ComputeDepth();
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index f3f3003b7e20a..55a23c3cca9be 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -927,7 +927,7 @@ public:
                            SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
                            unsigned Alignment, AtomicOrdering SuccessOrdering,
                            AtomicOrdering FailureOrdering,
-                           SynchronizationScope SynchScope);
+                           SyncScope::ID SSID);
   SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                            SDVTList VTs, SDValue Chain, SDValue Ptr,
                            SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
@@ -937,7 +937,7 @@ public:
   SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, const Value *PtrVal,
                     unsigned Alignment, AtomicOrdering Ordering,
-                    SynchronizationScope SynchScope);
+                    SyncScope::ID SSID);
   SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
 
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index d9f8af0e21d16..db42fb6c170c0 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1213,8 +1213,8 @@ public:
   /// Returns the Ranges that describes the dereference.
   const MDNode *getRanges() const { return MMO->getRanges(); }
 
-  /// Return the synchronization scope for this memory operation.
-  SynchronizationScope getSynchScope() const { return MMO->getSynchScope(); }
+  /// Returns the synchronization scope ID for this memory operation.
+  SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
 
   /// Return the atomic ordering requirements for this memory operation. For
   /// cmpxchg atomic operations, return the atomic ordering requirements when
@@ -1432,8 +1432,8 @@ public:
   int64_t getSExtValue() const { return Value->getSExtValue(); }
 
   bool isOne() const { return Value->isOne(); }
-  bool isNullValue() const { return Value->isNullValue(); }
-  bool isAllOnesValue() const { return Value->isAllOnesValue(); }
+  bool isNullValue() const { return Value->isZero(); }
+  bool isAllOnesValue() const { return Value->isMinusOne(); }
 
   bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/include/llvm/DebugInfo/CodeView/SymbolRecord.h
index 7941af8be8af2..cdfc1745cea5d 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolRecord.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolRecord.h
@@ -735,6 +735,10 @@ public:
   uint16_t VersionBackendQFE;
   StringRef Version;
 
+  void setLanguage(SourceLanguage Lang) {
+    Flags = CompileSym3Flags((uint32_t(Flags) & 0xFFFFFF00) | uint32_t(Lang));
+  }
+
   uint8_t getLanguage() const { return static_cast<uint32_t>(Flags) & 0xFF; }
   uint32_t getFlags() const { return static_cast<uint32_t>(Flags) & ~0xFF; }
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeIndex.h b/include/llvm/DebugInfo/CodeView/TypeIndex.h
index 10d51c2d6244f..e0c2226bdbd74 100644
--- a/include/llvm/DebugInfo/CodeView/TypeIndex.h
+++ b/include/llvm/DebugInfo/CodeView/TypeIndex.h
@@ -10,9 +10,11 @@
 #ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H
 #define LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H
 
+#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/Support/Endian.h"
 #include <cassert>
 #include <cinttypes>
+#include <functional>
 
 namespace llvm {
 
@@ -265,6 +267,23 @@ struct TypeIndexOffset {
 void printTypeIndex(ScopedPrinter &Printer, StringRef FieldName, TypeIndex TI,
                     TypeCollection &Types);
 }
-}
+
+template <> struct DenseMapInfo<codeview::TypeIndex> {
+  static inline codeview::TypeIndex getEmptyKey() {
+    return codeview::TypeIndex{DenseMapInfo<uint32_t>::getEmptyKey()};
+  }
+  static inline codeview::TypeIndex getTombstoneKey() {
+    return codeview::TypeIndex{DenseMapInfo<uint32_t>::getTombstoneKey()};
+  }
+  static unsigned getHashValue(const codeview::TypeIndex &TI) {
+    return DenseMapInfo<uint32_t>::getHashValue(TI.getIndex());
+  }
+  static bool isEqual(const codeview::TypeIndex &LHS,
+                      const codeview::TypeIndex &RHS) {
+    return LHS == RHS;
+  }
+};
+
+} // namespace llvm
 
 #endif
diff --git a/include/llvm/DebugInfo/DIContext.h b/include/llvm/DebugInfo/DIContext.h
index 4126e245ff139..936813dc6abc0 100644
--- a/include/llvm/DebugInfo/DIContext.h
+++ b/include/llvm/DebugInfo/DIContext.h
@@ -204,7 +204,9 @@ public:
   /// need to be consistent with the addresses used to query the DIContext and
   /// the output of this function should be deterministic, i.e. repeated calls with
   /// the same Sec should give the same address.
-  virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const = 0;
+  virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const {
+    return 0;
+  }
 
   /// If conveniently available, return the content of the given Section.
   ///
@@ -221,12 +223,28 @@ public:
     return false;
   }
 
+  // FIXME: This is untested and unused anywhere in the LLVM project, it's
+  // used/needed by Julia (an external project). It should have some coverage
+  // (at least tests, but ideally example functionality).
   /// Obtain a copy of this LoadedObjectInfo.
-  ///
-  /// The caller is responsible for deallocation once the copy is no longer required.
   virtual std::unique_ptr<LoadedObjectInfo> clone() const = 0;
 };
 
+template <typename Derived, typename Base = LoadedObjectInfo>
+struct LoadedObjectInfoHelper : Base {
+protected:
+  LoadedObjectInfoHelper(const LoadedObjectInfoHelper &) = default;
+  LoadedObjectInfoHelper() = default;
+
+public:
+  template <typename... Ts>
+  LoadedObjectInfoHelper(Ts &&... Args) : Base(std::forward<Ts>(Args)...) {}
+
+  std::unique_ptr<llvm::LoadedObjectInfo> clone() const override {
+    return llvm::make_unique<Derived>(static_cast<const Derived &>(*this));
+  }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_DEBUGINFO_DICONTEXT_H
diff --git a/include/llvm/DebugInfo/DWARF/DWARFContext.h b/include/llvm/DebugInfo/DWARF/DWARFContext.h
index 739aa1f9ee749..ee2e805050c01 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -226,11 +226,7 @@ public:
   virtual bool isLittleEndian() const = 0;
   virtual uint8_t getAddressSize() const = 0;
   virtual const DWARFSection &getInfoSection() = 0;
-
-  using TypeSectionMap = MapVector<object::SectionRef, DWARFSection,
-                                   std::map<object::SectionRef, unsigned>>;
-
-  virtual const TypeSectionMap &getTypesSections() = 0;
+  virtual void forEachTypesSections(function_ref<void(DWARFSection &)> F) = 0;
   virtual StringRef getAbbrevSection() = 0;
   virtual const DWARFSection &getLocSection() = 0;
   virtual StringRef getARangeSection() = 0;
@@ -252,7 +248,8 @@ public:
 
   // Sections for DWARF5 split dwarf proposal.
   virtual const DWARFSection &getInfoDWOSection() = 0;
-  virtual const TypeSectionMap &getTypesDWOSections() = 0;
+  virtual void
+  forEachTypesDWOSections(function_ref<void(DWARFSection &)> F) = 0;
   virtual StringRef getAbbrevDWOSection() = 0;
   virtual const DWARFSection &getLineDWOSection() = 0;
   virtual const DWARFSection &getLocDWOSection() = 0;
@@ -294,6 +291,9 @@ enum class ErrorPolicy { Halt, Continue };
 class DWARFContextInMemory : public DWARFContext {
   virtual void anchor();
 
+  using TypeSectionMap = MapVector<object::SectionRef, DWARFSection,
+                                   std::map<object::SectionRef, unsigned>>;
+
   StringRef FileName;
   bool IsLittleEndian;
   uint8_t AddressSize;
@@ -338,7 +338,8 @@ class DWARFContextInMemory : public DWARFContext {
 
   SmallVector<SmallString<32>, 4> UncompressedSections;
 
-  StringRef *MapSectionToMember(StringRef Name);
+  DWARFSection *mapNameToDWARFSection(StringRef Name);
+  StringRef *mapSectionToMember(StringRef Name);
 
   /// If Sec is compressed section, decompresses and updates its contents
   /// provided by Data. Otherwise leaves it unchanged.
@@ -362,7 +363,10 @@ public:
   bool isLittleEndian() const override { return IsLittleEndian; }
   uint8_t getAddressSize() const override { return AddressSize; }
   const DWARFSection &getInfoSection() override { return InfoSection; }
-  const TypeSectionMap &getTypesSections() override { return TypesSections; }
+  void forEachTypesSections(function_ref<void(DWARFSection &)> F) override {
+    for (auto &P : TypesSections)
+      F(P.second);
+  }
   StringRef getAbbrevSection() override { return AbbrevSection; }
   const DWARFSection &getLocSection() override { return LocSection; }
   StringRef getARangeSection() override { return ARangeSection; }
@@ -389,8 +393,9 @@ public:
   // Sections for DWARF5 split dwarf proposal.
   const DWARFSection &getInfoDWOSection() override { return InfoDWOSection; }
 
-  const TypeSectionMap &getTypesDWOSections() override {
-    return TypesDWOSections;
+  void forEachTypesDWOSections(function_ref<void(DWARFSection &)> F) override {
+    for (auto &P : TypesDWOSections)
+      F(P.second);
   }
 
   StringRef getAbbrevDWOSection() override { return AbbrevDWOSection; }
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
index e4cb1b24e30d0..c918a5d5e976d 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
@@ -47,6 +47,7 @@ public:
   DbiModuleDescriptorBuilder &
   operator=(const DbiModuleDescriptorBuilder &) = delete;
 
+  void setPdbFilePathNI(uint32_t NI);
   void setObjFileName(StringRef Name);
   void addSymbol(codeview::CVSymbol Symbol);
 
@@ -68,6 +69,10 @@ public:
 
   uint32_t calculateSerializedLength() const;
 
+  /// Return the offset within the module symbol stream of the next symbol
+  /// record passed to addSymbol. Add four to account for the signature.
+  uint32_t getNextSymbolOffset() const { return SymbolByteSize + 4; }
+
   void finalize();
   Error finalizeMsfLayout();
 
@@ -81,6 +86,7 @@ private:
   msf::MSFBuilder &MSF;
 
   uint32_t SymbolByteSize = 0;
+  uint32_t PdbFilePathNI = 0;
   std::string ModuleName;
   std::string ObjFileName;
   std::vector<std::string> SourceFiles;
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/include/llvm/DebugInfo/PDB/Native/DbiStream.h
index 3bf7907266564..4be113f28d6f5 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiStream.h
@@ -83,6 +83,8 @@ public:
   FixedStreamArray<SecMapEntry> getSectionMap() const;
   void visitSectionContributions(ISectionContribVisitor &Visitor) const;
 
+  Expected<StringRef> getECName(uint32_t NI) const;
+
 private:
   Error initializeSectionContributionData();
   Error initializeSectionHeadersData();
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
index 744411854181f..63eb34f0326af 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
@@ -15,6 +15,7 @@
 #include "llvm/Support/Error.h"
 
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
+#include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/Support/BinaryByteStream.h"
@@ -54,8 +55,13 @@ public:
   // Add given bytes as a new stream.
   Error addDbgStream(pdb::DbgHeaderType Type, ArrayRef<uint8_t> Data);
 
+  uint32_t addECName(StringRef Name);
+
   uint32_t calculateSerializedLength() const;
 
+  void setPublicsStreamIndex(uint32_t Index);
+  void setSymbolRecordStreamIndex(uint32_t Index);
+
   Expected<DbiModuleDescriptorBuilder &> addModuleInfo(StringRef ModuleName);
   Error addModuleSourceFile(StringRef Module, StringRef File);
   Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module, StringRef File);
@@ -75,7 +81,7 @@ public:
 private:
   struct DebugStream {
     ArrayRef<uint8_t> Data;
-    uint16_t StreamNumber = 0;
+    uint16_t StreamNumber = kInvalidStreamIndex;
   };
 
   Error finalize();
@@ -87,7 +93,6 @@ private:
   uint32_t calculateNamesBufferSize() const;
   uint32_t calculateDbgStreamsSize() const;
 
-  Error generateModiSubstream();
   Error generateFileInfoSubstream();
 
   msf::MSFBuilder &Msf;
@@ -100,6 +105,8 @@ private:
   uint16_t PdbDllRbld;
   uint16_t Flags;
   PDB_Machine MachineType;
+  uint32_t PublicsStreamIndex = kInvalidStreamIndex;
+  uint32_t SymRecordStreamIndex = kInvalidStreamIndex;
 
   const DbiStreamHeader *Header;
 
@@ -108,6 +115,7 @@ private:
 
   StringMap<uint32_t> SourceFileNames;
 
+  PDBStringTableBuilder ECNamesBuilder;
   WritableBinaryStreamRef NamesBuffer;
   MutableBinaryByteStream FileInfoBuffer;
   std::vector<SectionContrib> SectionContribs;
diff --git a/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h b/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
index 25f66240a6a24..17a82b7ce12db 100644
--- a/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
+++ b/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
@@ -44,7 +44,7 @@ public:
   bool get(StringRef Stream, uint32_t &StreamNo) const;
   void set(StringRef Stream, uint32_t StreamNo);
   void remove(StringRef Stream);
-
+  const StringMap<uint32_t> &getStringMap() const { return Mapping; }
   iterator_range<StringMapConstIterator<uint32_t>> entries() const;
 
 private:
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h
new file mode 100644
index 0000000000000..4f532c6e38299
--- /dev/null
+++ b/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h
@@ -0,0 +1,49 @@
+//===- NativeBuiltinSymbol.h -------------------------------------- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEBUILTINSYMBOL_H
+#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEBUILTINSYMBOL_H
+
+#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h"
+
+#include "llvm/DebugInfo/PDB/PDBTypes.h"
+
+namespace llvm {
+namespace pdb {
+
+class NativeSession;
+
+class NativeBuiltinSymbol : public NativeRawSymbol {
+public:
+  NativeBuiltinSymbol(NativeSession &PDBSession, SymIndexId Id,
+                      PDB_BuiltinType T, uint64_t L);
+  ~NativeBuiltinSymbol() override;
+
+  virtual std::unique_ptr<NativeRawSymbol> clone() const override;
+
+  void dump(raw_ostream &OS, int Indent) const override;
+
+  PDB_SymType getSymTag() const override;
+
+  PDB_BuiltinType getBuiltinType() const override;
+  bool isConstType() const override;
+  uint64_t getLength() const override;
+  bool isUnalignedType() const override;
+  bool isVolatileType() const override;
+
+protected:
+  NativeSession &Session;
+  PDB_BuiltinType Type;
+  uint64_t Length;
+};
+
+} // namespace pdb
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
index 1687737f0e7f9..bd5c09e5ff76d 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
@@ -18,7 +18,7 @@ namespace pdb {
 
 class NativeCompilandSymbol : public NativeRawSymbol {
 public:
-  NativeCompilandSymbol(NativeSession &Session, uint32_t SymbolId,
+  NativeCompilandSymbol(NativeSession &Session, SymIndexId SymbolId,
                         DbiModuleDescriptor MI);
 
   std::unique_ptr<NativeRawSymbol> clone() const override;
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
index 15bac78df191b..ddb7f811da384 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
@@ -18,7 +18,7 @@ namespace pdb {
 
 class NativeExeSymbol : public NativeRawSymbol {
 public:
-  NativeExeSymbol(NativeSession &Session, uint32_t SymbolId);
+  NativeExeSymbol(NativeSession &Session, SymIndexId SymbolId);
 
   std::unique_ptr<NativeRawSymbol> clone() const override;
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
index a24a972879d2c..66a9eae28e235 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
@@ -19,9 +19,11 @@ namespace pdb {
 
 class NativeSession;
 
+typedef uint32_t SymIndexId;
+
 class NativeRawSymbol : public IPDBRawSymbol {
 public:
-  NativeRawSymbol(NativeSession &PDBSession, uint32_t SymbolId);
+  NativeRawSymbol(NativeSession &PDBSession, SymIndexId SymbolId);
 
   virtual std::unique_ptr<NativeRawSymbol> clone() const = 0;
 
@@ -205,7 +207,7 @@ public:
 
 protected:
   NativeSession &Session;
-  uint32_t SymbolId;
+  SymIndexId SymbolId;
 };
 
 } // end namespace pdb
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/include/llvm/DebugInfo/PDB/Native/NativeSession.h
index dd40874dc5f26..b16ce231c349c 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeSession.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeSession.h
@@ -10,9 +10,13 @@
 #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVESESSION_H
 #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVESESSION_H
 
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
 #include "llvm/DebugInfo/PDB/IPDBSession.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
+#include "llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h"
 #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Error.h"
@@ -35,6 +39,8 @@ public:
   std::unique_ptr<PDBSymbolCompiland>
   createCompilandSymbol(DbiModuleDescriptor MI);
 
+  SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI);
+
   uint64_t getLoadAddress() const override;
   void setLoadAddress(uint64_t Address) override;
   std::unique_ptr<PDBSymbolExe> getGlobalScope() override;
@@ -77,6 +83,7 @@ private:
   std::unique_ptr<PDBFile> Pdb;
   std::unique_ptr<BumpPtrAllocator> Allocator;
   std::vector<std::unique_ptr<NativeRawSymbol>> SymbolCache;
+  DenseMap<codeview::TypeIndex, SymIndexId> TypeIndexToSymbolId;
 };
 }
 }
diff --git a/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
index cd7d3b0637933..2dc23f819d3bd 100644
--- a/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
@@ -31,11 +31,13 @@ class MSFBuilder;
 namespace pdb {
 class DbiStreamBuilder;
 class InfoStreamBuilder;
+class PublicsStreamBuilder;
 class TpiStreamBuilder;
 
 class PDBFileBuilder {
 public:
   explicit PDBFileBuilder(BumpPtrAllocator &Allocator);
+  ~PDBFileBuilder();
   PDBFileBuilder(const PDBFileBuilder &) = delete;
   PDBFileBuilder &operator=(const PDBFileBuilder &) = delete;
 
@@ -47,6 +49,7 @@ public:
   TpiStreamBuilder &getTpiBuilder();
   TpiStreamBuilder &getIpiBuilder();
   PDBStringTableBuilder &getStringTableBuilder();
+  PublicsStreamBuilder &getPublicsBuilder();
 
   Error commit(StringRef Filename);
 
@@ -61,6 +64,7 @@ private:
   std::unique_ptr<msf::MSFBuilder> Msf;
   std::unique_ptr<InfoStreamBuilder> Info;
   std::unique_ptr<DbiStreamBuilder> Dbi;
+  std::unique_ptr<PublicsStreamBuilder> Publics;
   std::unique_ptr<TpiStreamBuilder> Tpi;
   std::unique_ptr<TpiStreamBuilder> Ipi;
 
diff --git a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
index 86ef1136b41d7..29167c966d427 100644
--- a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
+++ b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
@@ -56,7 +56,6 @@ private:
   const PDBStringTableHeader *Header = nullptr;
   codeview::DebugStringTableSubsectionRef Strings;
   FixedStreamArray<support::ulittle32_t> IDs;
-  uint32_t ByteSize = 0;
   uint32_t NameCount = 0;
 };
 
diff --git a/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
index 4570c80c76d7c..9ace826bd8f71 100644
--- a/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
@@ -25,8 +25,6 @@ struct GSIHashHeader;
 class PDBFile;
 
 class PublicsStream {
-  struct HeaderInfo;
-
 public:
   PublicsStream(PDBFile &File, std::unique_ptr<msf::MappedBlockStream> Stream);
   ~PublicsStream();
@@ -65,7 +63,7 @@ private:
   FixedStreamArray<support::ulittle32_t> ThunkMap;
   FixedStreamArray<SectionOffset> SectionOffsets;
 
-  const HeaderInfo *Header;
+  const PublicsStreamHeader *Header;
   const GSIHashHeader *HashHdr;
 };
 }
diff --git a/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h
new file mode 100644
index 0000000000000..5ab57ebef53d4
--- /dev/null
+++ b/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h
@@ -0,0 +1,54 @@
+//===- PublicsStreamBuilder.h - PDB Publics Stream Creation -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBPUBLICSTREAMBUILDER_H
+#define LLVM_DEBUGINFO_PDB_RAW_PDBPUBLICSTREAMBUILDER_H
+
+#include "llvm/DebugInfo/PDB/Native/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Native/RawTypes.h"
+#include "llvm/Support/BinaryByteStream.h"
+#include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace msf {
+class MSFBuilder;
+}
+namespace pdb {
+class PublicsStream;
+struct PublicsStreamHeader;
+
+class PublicsStreamBuilder {
+public:
+  explicit PublicsStreamBuilder(msf::MSFBuilder &Msf);
+  ~PublicsStreamBuilder();
+
+  PublicsStreamBuilder(const PublicsStreamBuilder &) = delete;
+  PublicsStreamBuilder &operator=(const PublicsStreamBuilder &) = delete;
+
+  Error finalizeMsfLayout();
+  uint32_t calculateSerializedLength() const;
+
+  Error commit(BinaryStreamWriter &PublicsWriter);
+
+  uint32_t getStreamIndex() const { return StreamIdx; }
+  uint32_t getRecordStreamIdx() const { return RecordStreamIdx; }
+
+private:
+  uint32_t StreamIdx = kInvalidStreamIndex;
+  uint32_t RecordStreamIdx = kInvalidStreamIndex;
+  std::vector<PSHashRecord> HashRecords;
+  msf::MSFBuilder &Msf;
+};
+} // namespace pdb
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/DebugInfo/PDB/Native/RawTypes.h b/include/llvm/DebugInfo/PDB/Native/RawTypes.h
index 771272d6a47d1..a3cdd3f09a44e 100644
--- a/include/llvm/DebugInfo/PDB/Native/RawTypes.h
+++ b/include/llvm/DebugInfo/PDB/Native/RawTypes.h
@@ -255,6 +255,19 @@ struct ModuleInfoHeader {
   /// char ObjFileName[];
 };
 
+// This is PSGSIHDR struct defined in
+// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h
+struct PublicsStreamHeader {
+  support::ulittle32_t SymHash;
+  support::ulittle32_t AddrMap;
+  support::ulittle32_t NumThunks;
+  support::ulittle32_t SizeOfThunk;
+  support::ulittle16_t ISectThunkTable;
+  char Padding[2];
+  support::ulittle32_t OffThunkTable;
+  support::ulittle32_t NumSections;
+};
+
 /// Defines a 128-bit unique identifier.  This maps to a GUID on Windows, but
 /// is abstracted here for the purposes of non-Windows platforms that don't have
 /// the GUID structure defined.
diff --git a/include/llvm/ExecutionEngine/JITSymbol.h b/include/llvm/ExecutionEngine/JITSymbol.h
index f09e95fddb97e..4172f240ba392 100644
--- a/include/llvm/ExecutionEngine/JITSymbol.h
+++ b/include/llvm/ExecutionEngine/JITSymbol.h
@@ -21,6 +21,8 @@
 #include <functional>
 #include <string>
 
+#include "llvm/Support/Error.h"
+
 namespace llvm {
 
 class GlobalValue;
@@ -41,10 +43,11 @@ public:
 
   enum FlagNames : UnderlyingType {
     None = 0,
-    Weak = 1U << 0,
-    Common = 1U << 1,
-    Absolute = 1U << 2,
-    Exported = 1U << 3
+    HasError = 1U << 0,
+    Weak = 1U << 1,
+    Common = 1U << 2,
+    Absolute = 1U << 3,
+    Exported = 1U << 4
   };
 
   /// @brief Default-construct a JITSymbolFlags instance.
@@ -53,6 +56,11 @@ public:
   /// @brief Construct a JITSymbolFlags instance from the given flags.
   JITSymbolFlags(FlagNames Flags) : Flags(Flags) {}
 
+  /// @brief Return true if there was an error retrieving this symbol.
+  bool hasError() const {
+    return (Flags & HasError) == HasError;
+  }
+
   /// @brief Returns true is the Weak flag is set.
   bool isWeak() const {
     return (Flags & Weak) == Weak;
@@ -113,11 +121,17 @@ private:
 /// @brief Represents a symbol in the JIT.
 class JITSymbol {
 public:
-  using GetAddressFtor = std::function<JITTargetAddress()>;
+  using GetAddressFtor = std::function<Expected<JITTargetAddress>()>;
+
+  /// @brief Create a 'null' symbol, used to represent a "symbol not found"
+  ///        result from a successful (non-erroneous) lookup.
+  JITSymbol(std::nullptr_t)
+      : CachedAddr(0) {}
 
-  /// @brief Create a 'null' symbol that represents failure to find a symbol
-  ///        definition.
-  JITSymbol(std::nullptr_t) {}
+  /// @brief Create a JITSymbol representing an error in the symbol lookup
+  ///        process (e.g. a network failure during a remote lookup).
+  JITSymbol(Error Err)
+    : Err(std::move(Err)), Flags(JITSymbolFlags::HasError) {}
 
   /// @brief Create a symbol for a definition with a known address.
   JITSymbol(JITTargetAddress Addr, JITSymbolFlags Flags)
@@ -137,18 +151,59 @@ public:
   /// user can materialize the definition at any time by calling the getAddress
   /// method.
   JITSymbol(GetAddressFtor GetAddress, JITSymbolFlags Flags)
-      : GetAddress(std::move(GetAddress)), Flags(Flags) {}
+      : GetAddress(std::move(GetAddress)), CachedAddr(0), Flags(Flags) {}
+
+  JITSymbol(const JITSymbol&) = delete;
+  JITSymbol& operator=(const JITSymbol&) = delete;
+
+  JITSymbol(JITSymbol &&Other)
+    : GetAddress(std::move(Other.GetAddress)), Flags(std::move(Other.Flags)) {
+    if (Flags.hasError())
+      Err = std::move(Other.Err);
+    else
+      CachedAddr = std::move(Other.CachedAddr);
+  }
+
+  JITSymbol& operator=(JITSymbol &&Other) {
+    GetAddress = std::move(Other.GetAddress);
+    Flags = std::move(Other.Flags);
+    if (Flags.hasError())
+      Err = std::move(Other.Err);
+    else
+      CachedAddr = std::move(Other.CachedAddr);
+    return *this;
+  }
+
+  ~JITSymbol() {
+    if (Flags.hasError())
+      Err.~Error();
+    else
+      CachedAddr.~JITTargetAddress();
+  }
 
   /// @brief Returns true if the symbol exists, false otherwise.
-  explicit operator bool() const { return CachedAddr || GetAddress; }
+  explicit operator bool() const {
+    return !Flags.hasError() && (CachedAddr || GetAddress);
+  }
+
+  /// @brief Move the error field value out of this JITSymbol.
+  Error takeError() {
+    if (Flags.hasError())
+      return std::move(Err);
+    return Error::success();
+  }
 
   /// @brief Get the address of the symbol in the target address space. Returns
   ///        '0' if the symbol does not exist.
-  JITTargetAddress getAddress() {
+  Expected<JITTargetAddress> getAddress() {
+    assert(!Flags.hasError() && "getAddress called on error value");
     if (GetAddress) {
-      CachedAddr = GetAddress();
-      assert(CachedAddr && "Symbol could not be materialized.");
-      GetAddress = nullptr;
+      if (auto CachedAddrOrErr = GetAddress()) {
+        GetAddress = nullptr;
+        CachedAddr = *CachedAddrOrErr;
+        assert(CachedAddr && "Symbol could not be materialized.");
+      } else
+        return CachedAddrOrErr.takeError();
     }
     return CachedAddr;
   }
@@ -157,7 +212,10 @@ public:
 
 private:
   GetAddressFtor GetAddress;
-  JITTargetAddress CachedAddr = 0;
+  union {
+    JITTargetAddress CachedAddr;
+    Error Err;
+  };
   JITSymbolFlags Flags;
 };
 
diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
index 8ac1b6bca0a7a..c1acca386820f 100644
--- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
@@ -146,7 +146,7 @@ private:
                     std::unique_ptr<JITSymbolResolver>)>;
 
     struct SourceModuleEntry {
-      std::unique_ptr<ResourceOwner<Module>> SourceMod;
+      std::shared_ptr<Module> SourceMod;
       std::set<Function*> StubsToClone;
     };
 
@@ -154,7 +154,7 @@ private:
     using SourceModuleHandle = typename SourceModulesList::size_type;
 
     SourceModuleHandle
-    addSourceModule(std::unique_ptr<ResourceOwner<Module>> M) {
+    addSourceModule(std::shared_ptr<Module> M) {
       SourceModuleHandle H = SourceModules.size();
       SourceModules.push_back(SourceModuleEntry());
       SourceModules.back().SourceMod = std::move(M);
@@ -162,7 +162,7 @@ private:
     }
 
     Module& getSourceModule(SourceModuleHandle H) {
-      return SourceModules[H].SourceMod->getResource();
+      return *SourceModules[H].SourceMod;
     }
 
     std::set<Function*>& getStubsToClone(SourceModuleHandle H) {
@@ -176,19 +176,21 @@ private:
       for (auto BLH : BaseLayerHandles)
         if (auto Sym = BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly))
           return Sym;
+        else if (auto Err = Sym.takeError())
+          return std::move(Err);
       return nullptr;
     }
 
-    void removeModulesFromBaseLayer(BaseLayerT &BaseLayer) {
+    Error removeModulesFromBaseLayer(BaseLayerT &BaseLayer) {
       for (auto &BLH : BaseLayerHandles)
-        BaseLayer.removeModule(BLH);
+        if (auto Err = BaseLayer.removeModule(BLH))
+          return Err;
+      return Error::success();
     }
 
-    std::unique_ptr<JITSymbolResolver> ExternalSymbolResolver;
-    std::unique_ptr<ResourceOwner<RuntimeDyld::MemoryManager>> MemMgr;
+    std::shared_ptr<JITSymbolResolver> ExternalSymbolResolver;
     std::unique_ptr<IndirectStubsMgrT> StubsMgr;
     StaticGlobalRenamer StaticRenamer;
-    ModuleAdderFtor ModuleAdder;
     SourceModulesList SourceModules;
     std::vector<BaseLayerModuleHandleT> BaseLayerHandles;
   };
@@ -196,6 +198,7 @@ private:
   using LogicalDylibList = std::list<LogicalDylib>;
 
 public:
+
   /// @brief Handle to loaded module.
   using ModuleHandleT = typename LogicalDylibList::iterator;
 
@@ -217,48 +220,41 @@ public:
         CloneStubsIntoPartitions(CloneStubsIntoPartitions) {}
 
   ~CompileOnDemandLayer() {
+    // FIXME: Report error on log.
     while (!LogicalDylibs.empty())
-      removeModule(LogicalDylibs.begin());
+      consumeError(removeModule(LogicalDylibs.begin()));
   }
 
   /// @brief Add a module to the compile-on-demand layer.
-  template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-  ModuleHandleT addModule(std::shared_ptr<Module> M,
-                          MemoryManagerPtrT MemMgr,
-                          SymbolResolverPtrT Resolver) {
+  Expected<ModuleHandleT>
+  addModule(std::shared_ptr<Module> M,
+            std::shared_ptr<JITSymbolResolver> Resolver) {
 
     LogicalDylibs.push_back(LogicalDylib());
     auto &LD = LogicalDylibs.back();
     LD.ExternalSymbolResolver = std::move(Resolver);
     LD.StubsMgr = CreateIndirectStubsManager();
 
-    auto &MemMgrRef = *MemMgr;
-    LD.MemMgr = wrapOwnership<RuntimeDyld::MemoryManager>(std::move(MemMgr));
-
-    LD.ModuleAdder =
-      [&MemMgrRef](BaseLayerT &B, std::unique_ptr<Module> M,
-                   std::unique_ptr<JITSymbolResolver> R) {
-        return B.addModule(std::move(M), &MemMgrRef, std::move(R));
-      };
-
     // Process each of the modules in this module set.
-    addLogicalModule(LogicalDylibs.back(), std::move(M));
+    if (auto Err = addLogicalModule(LD, std::move(M)))
+      return std::move(Err);
 
     return std::prev(LogicalDylibs.end());
   }
 
   /// @brief Add extra modules to an existing logical module.
-  void addExtraModule(ModuleHandleT H, std::shared_ptr<Module> M) {
-    addLogicalModule(*H, std::move(M));
+  Error addExtraModule(ModuleHandleT H, std::shared_ptr<Module> M) {
+    return addLogicalModule(*H, std::move(M));
   }
 
   /// @brief Remove the module represented by the given handle.
   ///
   ///   This will remove all modules in the layers below that were derived from
   /// the module represented by H.
-  void removeModule(ModuleHandleT H) {
-    H->removeModulesFromBaseLayer(BaseLayer);
+  Error removeModule(ModuleHandleT H) {
+    auto Err = H->removeModulesFromBaseLayer(BaseLayer);
     LogicalDylibs.erase(H);
+    return Err;
   }
 
   /// @brief Search for the given named symbol.
@@ -272,6 +268,8 @@ public:
         return Sym;
       if (auto Sym = findSymbolIn(LDI, Name, ExportedSymbolsOnly))
         return Sym;
+      else if (auto Err = Sym.takeError())
+        return std::move(Err);
     }
     return BaseLayer.findSymbol(Name, ExportedSymbolsOnly);
   }
@@ -309,8 +307,9 @@ public:
   }
 
 private:
-  template <typename ModulePtrT>
-  void addLogicalModule(LogicalDylib &LD, ModulePtrT SrcMPtr) {
+
+  Error addLogicalModule(LogicalDylib &LD, std::shared_ptr<Module> SrcMPtr) {
+
     // Rename all static functions / globals to $static.X :
     // This will unique the names across all modules in the logical dylib,
     // simplifying symbol lookup.
@@ -322,7 +321,7 @@ private:
 
     // Create a logical module handle for SrcM within the logical dylib.
     Module &SrcM = *SrcMPtr;
-    auto LMId = LD.addSourceModule(wrapOwnership<Module>(std::move(SrcMPtr)));
+    auto LMId = LD.addSourceModule(std::move(SrcMPtr));
 
     // Create stub functions.
     const DataLayout &DL = SrcM.getDataLayout();
@@ -335,9 +334,12 @@ private:
 
         // Skip weak functions for which we already have definitions.
         auto MangledName = mangle(F.getName(), DL);
-        if (F.hasWeakLinkage() || F.hasLinkOnceLinkage())
+        if (F.hasWeakLinkage() || F.hasLinkOnceLinkage()) {
           if (auto Sym = LD.findSymbol(BaseLayer, MangledName, false))
             continue;
+          else if (auto Err = Sym.takeError())
+            return std::move(Err);
+        }
 
         // Record all functions defined by this module.
         if (CloneStubsIntoPartitions)
@@ -350,9 +352,15 @@ private:
         StubInits[MangledName] =
           std::make_pair(CCInfo.getAddress(),
                          JITSymbolFlags::fromGlobalValue(F));
-        CCInfo.setCompileAction([this, &LD, LMId, &F]() {
-          return this->extractAndCompile(LD, LMId, F);
-        });
+        CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress {
+            if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F))
+              return *FnImplAddrOrErr;
+            else {
+              // FIXME: Report error, return to 'abort' or something similar.
+              consumeError(FnImplAddrOrErr.takeError());
+              return 0;
+            }
+          });
       }
 
       auto EC = LD.StubsMgr->createStubs(StubInits);
@@ -367,7 +375,7 @@ private:
     // empty globals module.
     if (SrcM.global_empty() && SrcM.alias_empty() &&
         !SrcM.getModuleFlagsMetadata())
-      return;
+      return Error::success();
 
     // Create the GlobalValues module.
     auto GVsM = llvm::make_unique<Module>((SrcM.getName() + ".globals").str(),
@@ -393,8 +401,9 @@ private:
 
     // Initializers may refer to functions declared (but not defined) in this
     // module. Build a materializer to clone decls on demand.
+    Error MaterializerErrors = Error::success();
     auto Materializer = createLambdaMaterializer(
-      [&LD, &GVsM](Value *V) -> Value* {
+      [&LD, &GVsM, &MaterializerErrors](Value *V) -> Value* {
         if (auto *F = dyn_cast<Function>(V)) {
           // Decls in the original module just get cloned.
           if (F->isDeclaration())
@@ -405,13 +414,24 @@ private:
           // instead.
           const DataLayout &DL = GVsM->getDataLayout();
           std::string FName = mangle(F->getName(), DL);
-          auto StubSym = LD.StubsMgr->findStub(FName, false);
           unsigned PtrBitWidth = DL.getPointerTypeSizeInBits(F->getType());
-          ConstantInt *StubAddr =
-            ConstantInt::get(GVsM->getContext(),
-                             APInt(PtrBitWidth, StubSym.getAddress()));
+          JITTargetAddress StubAddr = 0;
+
+          // Get the address for the stub. If we encounter an error while
+          // doing so, stash it in the MaterializerErrors variable and use a
+          // null address as a placeholder.
+          if (auto StubSym = LD.StubsMgr->findStub(FName, false)) {
+            if (auto StubAddrOrErr = StubSym.getAddress())
+              StubAddr = *StubAddrOrErr;
+            else
+              MaterializerErrors = joinErrors(std::move(MaterializerErrors),
+                                              StubAddrOrErr.takeError());
+          }
+
+          ConstantInt *StubAddrCI =
+            ConstantInt::get(GVsM->getContext(), APInt(PtrBitWidth, StubAddr));
           Constant *Init = ConstantExpr::getCast(Instruction::IntToPtr,
-                                                 StubAddr, F->getType());
+                                                 StubAddrCI, F->getType());
           return GlobalAlias::create(F->getFunctionType(),
                                      F->getType()->getAddressSpace(),
                                      F->getLinkage(), F->getName(),
@@ -435,22 +455,31 @@ private:
       NewA->setAliasee(cast<Constant>(Init));
     }
 
+    if (MaterializerErrors)
+      return MaterializerErrors;
+
     // Build a resolver for the globals module and add it to the base layer.
     auto GVsResolver = createLambdaResolver(
-        [this, &LD](const std::string &Name) {
+        [this, &LD](const std::string &Name) -> JITSymbol {
           if (auto Sym = LD.StubsMgr->findStub(Name, false))
             return Sym;
           if (auto Sym = LD.findSymbol(BaseLayer, Name, false))
             return Sym;
+          else if (auto Err = Sym.takeError())
+            return std::move(Err);
           return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name);
         },
         [&LD](const std::string &Name) {
           return LD.ExternalSymbolResolver->findSymbol(Name);
         });
 
-    auto GVsH = LD.ModuleAdder(BaseLayer, std::move(GVsM),
-                               std::move(GVsResolver));
-    LD.BaseLayerHandles.push_back(GVsH);
+    if (auto GVsHOrErr =
+          BaseLayer.addModule(std::move(GVsM), std::move(GVsResolver)))
+      LD.BaseLayerHandles.push_back(*GVsHOrErr);
+    else
+      return GVsHOrErr.takeError();
+
+    return Error::success();
   }
 
   static std::string mangle(StringRef Name, const DataLayout &DL) {
@@ -462,7 +491,7 @@ private:
     return MangledName;
   }
 
-  JITTargetAddress
+  Expected<JITTargetAddress>
   extractAndCompile(LogicalDylib &LD,
                     typename LogicalDylib::SourceModuleHandle LMId,
                     Function &F) {
@@ -475,34 +504,42 @@ private:
     // Grab the name of the function being called here.
     std::string CalledFnName = mangle(F.getName(), SrcM.getDataLayout());
 
-    auto Part = Partition(F);
-    auto PartH = emitPartition(LD, LMId, Part);
-
     JITTargetAddress CalledAddr = 0;
-    for (auto *SubF : Part) {
-      std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout());
-      auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false);
-      assert(FnBodySym && "Couldn't find function body.");
-
-      JITTargetAddress FnBodyAddr = FnBodySym.getAddress();
-
-      // If this is the function we're calling record the address so we can
-      // return it from this function.
-      if (SubF == &F)
-        CalledAddr = FnBodyAddr;
-
-      // Update the function body pointer for the stub.
-      if (auto EC = LD.StubsMgr->updatePointer(FnName, FnBodyAddr))
-        return 0;
-    }
+    auto Part = Partition(F);
+    if (auto PartHOrErr = emitPartition(LD, LMId, Part)) {
+      auto &PartH = *PartHOrErr;
+      for (auto *SubF : Part) {
+        std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout());
+        if (auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false)) {
+          if (auto FnBodyAddrOrErr = FnBodySym.getAddress()) {
+            JITTargetAddress FnBodyAddr = *FnBodyAddrOrErr;
+
+            // If this is the function we're calling record the address so we can
+            // return it from this function.
+            if (SubF == &F)
+              CalledAddr = FnBodyAddr;
+
+            // Update the function body pointer for the stub.
+            if (auto EC = LD.StubsMgr->updatePointer(FnName, FnBodyAddr))
+              return 0;
+
+          } else
+            return FnBodyAddrOrErr.takeError();
+        } else if (auto Err = FnBodySym.takeError())
+          return std::move(Err);
+        else
+          llvm_unreachable("Function not emitted for partition");
+      }
 
-    LD.BaseLayerHandles.push_back(PartH);
+      LD.BaseLayerHandles.push_back(PartH);
+    } else
+      return PartHOrErr.takeError();
 
     return CalledAddr;
   }
 
   template <typename PartitionT>
-  BaseLayerModuleHandleT
+  Expected<BaseLayerModuleHandleT>
   emitPartition(LogicalDylib &LD,
                 typename LogicalDylib::SourceModuleHandle LMId,
                 const PartitionT &Part) {
@@ -566,16 +603,18 @@ private:
 
     // Create memory manager and symbol resolver.
     auto Resolver = createLambdaResolver(
-        [this, &LD](const std::string &Name) {
+        [this, &LD](const std::string &Name) -> JITSymbol {
           if (auto Sym = LD.findSymbol(BaseLayer, Name, false))
             return Sym;
+          else if (auto Err = Sym.takeError())
+            return std::move(Err);
           return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name);
         },
         [&LD](const std::string &Name) {
           return LD.ExternalSymbolResolver->findSymbol(Name);
         });
 
-    return LD.ModuleAdder(BaseLayer, std::move(M), std::move(Resolver));
+    return BaseLayer.addModule(std::move(M), std::move(Resolver));
   }
 
   BaseLayerT &BaseLayer;
diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index bf8cca406844d..d9b45c6a1e297 100644
--- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -17,6 +17,8 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/RuntimeDyld.h"
+#include "llvm/ExecutionEngine/Orc/OrcError.h"
 #include <algorithm>
 #include <cstdint>
 #include <string>
@@ -99,19 +101,24 @@ public:
 
   /// @brief Run the recorded constructors/destructors through the given JIT
   ///        layer.
-  bool runViaLayer(JITLayerT &JITLayer) const {
+  Error runViaLayer(JITLayerT &JITLayer) const {
     using CtorDtorTy = void (*)();
 
-    bool Error = false;
     for (const auto &CtorDtorName : CtorDtorNames)
       if (auto CtorDtorSym = JITLayer.findSymbolIn(H, CtorDtorName, false)) {
-        CtorDtorTy CtorDtor =
-          reinterpret_cast<CtorDtorTy>(
-            static_cast<uintptr_t>(CtorDtorSym.getAddress()));
-        CtorDtor();
-      } else
-        Error = true;
-    return !Error;
+        if (auto AddrOrErr = CtorDtorSym.getAddress()) {
+          CtorDtorTy CtorDtor =
+            reinterpret_cast<CtorDtorTy>(static_cast<uintptr_t>(*AddrOrErr));
+          CtorDtor();
+        } else
+          return AddrOrErr.takeError();
+      } else {
+        if (auto Err = CtorDtorSym.takeError())
+          return Err;
+        else
+          return make_error<JITSymbolNotFound>(CtorDtorName);
+      }
+    return Error::success();
   }
 
 private:
diff --git a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h
index d582e9a33241f..ff54ef625ebb7 100644
--- a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h
@@ -17,9 +17,14 @@
 
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include <map>
+#include <memory>
 #include <string>
 
 namespace llvm {
+
+class Module;
+class JITSymbolResolver;
+
 namespace orc {
 
 /// @brief Global mapping layer.
@@ -32,25 +37,22 @@ namespace orc {
 template <typename BaseLayerT>
 class GlobalMappingLayer {
 public:
-  /// @brief Handle to a set of added modules.
-  using ModuleSetHandleT = typename BaseLayerT::ModuleSetHandleT;
+
+  /// @brief Handle to an added module.
+  using ModuleHandleT = typename BaseLayerT::ModuleHandleT;
 
   /// @brief Construct an GlobalMappingLayer with the given BaseLayer
   GlobalMappingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {}
 
-  /// @brief Add the given module set to the JIT.
+  /// @brief Add the given module to the JIT.
   /// @return A handle for the added modules.
-  template <typename ModuleSetT, typename MemoryManagerPtrT,
-            typename SymbolResolverPtrT>
-  ModuleSetHandleT addModuleSet(ModuleSetT Ms,
-                                MemoryManagerPtrT MemMgr,
-                                SymbolResolverPtrT Resolver) {
-    return BaseLayer.addModuleSet(std::move(Ms), std::move(MemMgr),
-                                  std::move(Resolver));
+  ModuleHandleT addModule(std::shared_ptr<Module> M,
+                          std::shared_ptr<JITSymbolResolver> Resolver) {
+    return BaseLayer.addModule(std::move(M), std::move(Resolver));
   }
 
   /// @brief Remove the module set associated with the handle H.
-  void removeModuleSet(ModuleSetHandleT H) { BaseLayer.removeModuleSet(H); }
+  void removeModule(ModuleHandleT H) { BaseLayer.removeModule(H); }
 
   /// @brief Manually set the address to return for the given symbol.
   void setGlobalMapping(const std::string &Name, JITTargetAddress Addr) {
@@ -78,15 +80,15 @@ public:
     return BaseLayer.findSymbol(Name, ExportedSymbolsOnly);
   }
 
-  /// @brief Get the address of the given symbol in the context of the set of
-  ///        modules represented by the handle H. This call is forwarded to the
+  /// @brief Get the address of the given symbol in the context of the of the
+  ///        module represented by the handle H. This call is forwarded to the
   ///        base layer's implementation.
-  /// @param H The handle for the module set to search in.
+  /// @param H The handle for the module to search in.
   /// @param Name The name of the symbol to search for.
   /// @param ExportedSymbolsOnly If true, search only for exported symbols.
   /// @return A handle for the given named symbol, if it is found in the
-  ///         given module set.
-  JITSymbol findSymbolIn(ModuleSetHandleT H, const std::string &Name,
+  ///         given module.
+  JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name,
                          bool ExportedSymbolsOnly) {
     return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly);
   }
@@ -94,7 +96,7 @@ public:
   /// @brief Immediately emit and finalize the module set represented by the
   ///        given handle.
   /// @param H Handle for module set to emit/finalize.
-  void emitAndFinalize(ModuleSetHandleT H) {
+  void emitAndFinalize(ModuleHandleT H) {
     BaseLayer.emitAndFinalize(H);
   }
 
diff --git a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
index 99ccd4d221a5e..fadd334bed0f1 100644
--- a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
@@ -50,18 +50,18 @@ public:
   ///        along with the given memory manager and symbol resolver.
   ///
   /// @return A handle for the added module.
-  template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-  ModuleHandleT addModule(std::shared_ptr<Module> M,
-                          MemoryManagerPtrT MemMgr,
-                          SymbolResolverPtrT Resolver) {
+  Expected<ModuleHandleT>
+  addModule(std::shared_ptr<Module> M,
+            std::shared_ptr<JITSymbolResolver> Resolver) {
     using CompileResult = decltype(Compile(*M));
     auto Obj = std::make_shared<CompileResult>(Compile(*M));
-    return BaseLayer.addObject(std::move(Obj), std::move(MemMgr),
-                               std::move(Resolver));
+    return BaseLayer.addObject(std::move(Obj), std::move(Resolver));
   }
 
   /// @brief Remove the module associated with the handle H.
-  void removeModule(ModuleHandleT H) { BaseLayer.removeObject(H); }
+  Error removeModule(ModuleHandleT H) {
+    return BaseLayer.removeObject(H);
+  }
 
   /// @brief Search for the given named symbol.
   /// @param Name The name of the symbol to search for.
@@ -87,8 +87,8 @@ public:
   /// @brief Immediately emit and finalize the module represented by the given
   ///        handle.
   /// @param H Handle for module to emit/finalize.
-  void emitAndFinalize(ModuleHandleT H) {
-    BaseLayer.emitAndFinalize(H);
+  Error emitAndFinalize(ModuleHandleT H) {
+    return BaseLayer.emitAndFinalize(H);
   }
 
 private:
diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
index cf6556a33bbd6..476061afda599 100644
--- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
@@ -42,16 +42,14 @@ public:
   ///        the layer below, along with the memory manager and symbol resolver.
   ///
   /// @return A handle for the added modules.
-  template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-  ModuleHandleT addModule(std::shared_ptr<Module> M,
-                          MemoryManagerPtrT MemMgr,
-                          SymbolResolverPtrT Resolver) {
-    return BaseLayer.addModule(Transform(std::move(M)), std::move(MemMgr),
-                               std::move(Resolver));
+  Expected<ModuleHandleT>
+  addModule(std::shared_ptr<Module> M,
+            std::shared_ptr<JITSymbolResolver> Resolver) {
+    return BaseLayer.addModule(Transform(std::move(M)), std::move(Resolver));
   }
 
   /// @brief Remove the module associated with the handle H.
-  void removeModule(ModuleHandleT H) { BaseLayer.removeModule(H); }
+  Error removeModule(ModuleHandleT H) { return BaseLayer.removeModule(H); }
 
   /// @brief Search for the given named symbol.
   /// @param Name The name of the symbol to search for.
@@ -77,8 +75,8 @@ public:
   /// @brief Immediately emit and finalize the module represented by the given
   ///        handle.
   /// @param H Handle for module to emit/finalize.
-  void emitAndFinalize(ModuleHandleT H) {
-    BaseLayer.emitAndFinalize(H);
+  Error emitAndFinalize(ModuleHandleT H) {
+    return BaseLayer.emitAndFinalize(H);
   }
 
   /// @brief Access the transform functor directly.
diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h
index 6868640d38e80..228392ae0d4ac 100644
--- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h
+++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h
@@ -45,7 +45,7 @@ private:
 
 template <typename DylibLookupFtorT,
           typename ExternalLookupFtorT>
-std::unique_ptr<LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>>
+std::shared_ptr<LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>>
 createLambdaResolver(DylibLookupFtorT DylibLookupFtor,
                      ExternalLookupFtorT ExternalLookupFtor) {
   using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>;
diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
index 38769aac12afe..6c951fab61855 100644
--- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
@@ -46,8 +46,9 @@ public:
 private:
   class EmissionDeferredModule {
   public:
-    EmissionDeferredModule() = default;
-    virtual ~EmissionDeferredModule() = default;
+    EmissionDeferredModule(std::shared_ptr<Module> M,
+                           std::shared_ptr<JITSymbolResolver> Resolver)
+      : M(std::move(M)), Resolver(std::move(Resolver)) {}
 
     JITSymbol find(StringRef Name, bool ExportedSymbolsOnly, BaseLayerT &B) {
       switch (EmitState) {
@@ -59,16 +60,24 @@ private:
           std::string PName = Name;
           JITSymbolFlags Flags = JITSymbolFlags::fromGlobalValue(*GV);
           auto GetAddress =
-            [this, ExportedSymbolsOnly, PName, &B]() -> JITTargetAddress {
+            [this, ExportedSymbolsOnly, PName, &B]() -> Expected<JITTargetAddress> {
               if (this->EmitState == Emitting)
                 return 0;
               else if (this->EmitState == NotEmitted) {
                 this->EmitState = Emitting;
-                Handle = this->emitToBaseLayer(B);
+                if (auto HandleOrErr = this->emitToBaseLayer(B))
+                  Handle = std::move(*HandleOrErr);
+                else
+                  return HandleOrErr.takeError();
                 this->EmitState = Emitted;
               }
-              auto Sym = B.findSymbolIn(Handle, PName, ExportedSymbolsOnly);
-              return Sym.getAddress();
+              if (auto Sym = B.findSymbolIn(Handle, PName, ExportedSymbolsOnly))
+                return Sym.getAddress();
+              else if (auto Err = Sym.takeError())
+                return std::move(Err);
+              else
+                llvm_unreachable("Successful symbol lookup should return "
+                                 "definition address here");
           };
           return JITSymbol(std::move(GetAddress), Flags);
         } else
@@ -101,33 +110,10 @@ private:
       BaseLayer.emitAndFinalize(Handle);
     }
 
-    template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-    static std::unique_ptr<EmissionDeferredModule>
-    create(BaseLayerT &B, std::shared_ptr<Module> M, MemoryManagerPtrT MemMgr,
-           SymbolResolverPtrT Resolver);
-
-  protected:
-    virtual const GlobalValue* searchGVs(StringRef Name,
-                                         bool ExportedSymbolsOnly) const = 0;
-    virtual BaseLayerHandleT emitToBaseLayer(BaseLayerT &BaseLayer) = 0;
-
   private:
-    enum { NotEmitted, Emitting, Emitted } EmitState = NotEmitted;
-    BaseLayerHandleT Handle;
-  };
-
-  template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-  class EmissionDeferredModuleImpl : public EmissionDeferredModule {
-  public:
-    EmissionDeferredModuleImpl(std::shared_ptr<Module> M,
-                               MemoryManagerPtrT MemMgr,
-                               SymbolResolverPtrT Resolver)
-        : M(std::move(M)), MemMgr(std::move(MemMgr)),
-          Resolver(std::move(Resolver)) {}
 
-  protected:
     const GlobalValue* searchGVs(StringRef Name,
-                                 bool ExportedSymbolsOnly) const override {
+                                 bool ExportedSymbolsOnly) const {
       // FIXME: We could clean all this up if we had a way to reliably demangle
       //        names: We could just demangle name and search, rather than
       //        mangling everything else.
@@ -149,15 +135,13 @@ private:
       return buildMangledSymbols(Name, ExportedSymbolsOnly);
     }
 
-    BaseLayerHandleT emitToBaseLayer(BaseLayerT &BaseLayer) override {
+    Expected<BaseLayerHandleT> emitToBaseLayer(BaseLayerT &BaseLayer) {
       // We don't need the mangled names set any more: Once we've emitted this
       // to the base layer we'll just look for symbols there.
       MangledSymbols.reset();
-      return BaseLayer.addModule(std::move(M), std::move(MemMgr),
-                                 std::move(Resolver));
+      return BaseLayer.addModule(std::move(M), std::move(Resolver));
     }
 
-  private:
     // If the mangled name of the given GlobalValue matches the given search
     // name (and its visibility conforms to the ExportedSymbolsOnly flag) then
     // return the symbol. Otherwise, add the mangled name to the Names map and
@@ -207,9 +191,10 @@ private:
       return nullptr;
     }
 
+    enum { NotEmitted, Emitting, Emitted } EmitState = NotEmitted;
+    BaseLayerHandleT Handle;
     std::shared_ptr<Module> M;
-    MemoryManagerPtrT MemMgr;
-    SymbolResolverPtrT Resolver;
+    std::shared_ptr<JITSymbolResolver> Resolver;
     mutable std::unique_ptr<StringMap<const GlobalValue*>> MangledSymbols;
   };
 
@@ -219,6 +204,7 @@ private:
   ModuleListT ModuleList;
 
 public:
+
   /// @brief Handle to a loaded module.
   using ModuleHandleT = typename ModuleListT::iterator;
 
@@ -226,24 +212,23 @@ public:
   LazyEmittingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {}
 
   /// @brief Add the given module to the lazy emitting layer.
-  template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-  ModuleHandleT addModule(std::shared_ptr<Module> M,
-                          MemoryManagerPtrT MemMgr,
-                          SymbolResolverPtrT Resolver) {
+  Expected<ModuleHandleT>
+  addModule(std::shared_ptr<Module> M,
+            std::shared_ptr<JITSymbolResolver> Resolver) {
     return ModuleList.insert(
         ModuleList.end(),
-        EmissionDeferredModule::create(BaseLayer, std::move(M),
-                                       std::move(MemMgr),
-                                       std::move(Resolver)));
+        llvm::make_unique<EmissionDeferredModule>(std::move(M),
+                                                  std::move(Resolver)));
   }
 
   /// @brief Remove the module represented by the given handle.
   ///
   ///   This method will free the memory associated with the given module, both
   /// in this layer, and the base layer.
-  void removeModule(ModuleHandleT H) {
+  Error removeModule(ModuleHandleT H) {
     (*H)->removeModuleFromBaseLayer(BaseLayer);
     ModuleList.erase(H);
+    return Error::success();
   }
 
   /// @brief Search for the given named symbol.
@@ -276,22 +261,11 @@ public:
   /// @brief Immediately emit and finalize the module represented by the given
   ///        handle.
   /// @param H Handle for module to emit/finalize.
-  void emitAndFinalize(ModuleHandleT H) {
-    (*H)->emitAndFinalize(BaseLayer);
+  Error emitAndFinalize(ModuleHandleT H) {
+    return (*H)->emitAndFinalize(BaseLayer);
   }
 };
 
-template <typename BaseLayerT>
-template <typename MemoryManagerPtrT, typename SymbolResolverPtrT>
-std::unique_ptr<typename LazyEmittingLayer<BaseLayerT>::EmissionDeferredModule>
-LazyEmittingLayer<BaseLayerT>::EmissionDeferredModule::create(
-    BaseLayerT &B, std::shared_ptr<Module> M, MemoryManagerPtrT MemMgr,
-    SymbolResolverPtrT Resolver) {
-  using EDS = EmissionDeferredModuleImpl<MemoryManagerPtrT, SymbolResolverPtrT>;
-  return llvm::make_unique<EDS>(std::move(M), std::move(MemMgr),
-                                std::move(Resolver));
-}
-
 } // end namespace orc
 } // end namespace llvm
 
diff --git a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
index c41c1233c0d96..cb47e7520b1ab 100644
--- a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include <algorithm>
+#include <memory>
 #include <string>
 
 namespace llvm {
@@ -42,16 +43,14 @@ public:
   ///        memory manager and symbol resolver.
   ///
   /// @return A handle for the added objects.
-  template <typename ObjPtrT, typename MemoryManagerPtrT,
-            typename SymbolResolverPtrT>
-  ObjHandleT addObject(ObjPtrT Obj, MemoryManagerPtrT MemMgr,
-                       SymbolResolverPtrT Resolver) {
-    return BaseLayer.addObject(Transform(std::move(Obj)), std::move(MemMgr),
-                               std::move(Resolver));
+  template <typename ObjectPtr>
+  Expected<ObjHandleT> addObject(ObjectPtr Obj,
+                                 std::shared_ptr<JITSymbolResolver> Resolver) {
+    return BaseLayer.addObject(Transform(std::move(Obj)), std::move(Resolver));
   }
 
   /// @brief Remove the object set associated with the handle H.
-  void removeObject(ObjHandleT H) { BaseLayer.removeObject(H); }
+  Error removeObject(ObjHandleT H) { return BaseLayer.removeObject(H); }
 
   /// @brief Search for the given named symbol.
   /// @param Name The name of the symbol to search for.
@@ -77,7 +76,9 @@ public:
   /// @brief Immediately emit and finalize the object set represented by the
   ///        given handle.
   /// @param H Handle for object set to emit/finalize.
-  void emitAndFinalize(ObjHandleT H) { BaseLayer.emitAndFinalize(H); }
+  Error emitAndFinalize(ObjHandleT H) {
+    return BaseLayer.emitAndFinalize(H);
+  }
 
   /// @brief Map section addresses for the objects associated with the handle H.
   void mapSectionAddress(ObjHandleT H, const void *LocalAddress,
diff --git a/include/llvm/ExecutionEngine/Orc/OrcError.h b/include/llvm/ExecutionEngine/Orc/OrcError.h
index cbb40fad02230..e6374b70967ae 100644
--- a/include/llvm/ExecutionEngine/Orc/OrcError.h
+++ b/include/llvm/ExecutionEngine/Orc/OrcError.h
@@ -22,7 +22,8 @@ namespace orc {
 
 enum class OrcErrorCode : int {
   // RPC Errors
-  RemoteAllocatorDoesNotExist = 1,
+  JITSymbolNotFound = 1,
+  RemoteAllocatorDoesNotExist,
   RemoteAllocatorIdAlreadyInUse,
   RemoteMProtectAddrUnrecognized,
   RemoteIndirectStubsOwnerDoesNotExist,
@@ -37,6 +38,18 @@ enum class OrcErrorCode : int {
 
 std::error_code orcError(OrcErrorCode ErrCode);
 
+class JITSymbolNotFound : public ErrorInfo<JITSymbolNotFound> {
+public:
+  static char ID;
+
+  JITSymbolNotFound(std::string SymbolName);
+  std::error_code convertToErrorCode() const override;
+  void log(raw_ostream &OS) const override;
+  const std::string &getSymbolName() const;
+private:
+  std::string SymbolName;
+};
+
 } // End namespace orc.
 } // End namespace llvm.
 
diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
index 66ad36be01c8d..e1016ef95f0c2 100644
--- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
@@ -228,13 +228,20 @@ private:
 
 public:
 
+  /// @brief Functor for creating memory managers.
+  using MemoryManagerGetter =
+    std::function<std::shared_ptr<RuntimeDyld::MemoryManager>()>;
+
   /// @brief Construct an ObjectLinkingLayer with the given NotifyLoaded,
   ///        and NotifyFinalized functors.
   RTDyldObjectLinkingLayer(
+      MemoryManagerGetter GetMemMgr,
       NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(),
       NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor())
-      : NotifyLoaded(std::move(NotifyLoaded)),
-        NotifyFinalized(std::move(NotifyFinalized)) {}
+      : GetMemMgr(GetMemMgr),
+        NotifyLoaded(std::move(NotifyLoaded)),
+        NotifyFinalized(std::move(NotifyFinalized)),
+        ProcessAllSections(false) {}
 
   /// @brief Set the 'ProcessAllSections' flag.
   ///
@@ -251,12 +258,8 @@ public:
   ///
   /// @return A handle that can be used to refer to the loaded objects (for 
   ///         symbol searching, finalization, freeing memory, etc.).
-  template <typename MemoryManagerPtrT,
-            typename SymbolResolverPtrT>
-  ObjHandleT addObject(ObjectPtr Obj,
-                       MemoryManagerPtrT MemMgr,
-                       SymbolResolverPtrT Resolver) {
-
+  Expected<ObjHandleT> addObject(ObjectPtr Obj,
+                                 std::shared_ptr<JITSymbolResolver> Resolver) {
     auto Finalizer = [&](ObjHandleT H, RuntimeDyld &RTDyld,
                          const ObjectPtr &ObjToLoad,
                          std::function<void()> LOSHandleLoad) {
@@ -275,8 +278,9 @@ public:
     };
 
     auto LO =
-      createLinkedObject(std::move(Obj), std::move(MemMgr), std::move(Resolver),
-                         std::move(Finalizer), ProcessAllSections);
+      createLinkedObject(std::move(Obj), GetMemMgr(),
+                         std::move(Resolver), std::move(Finalizer),
+                         ProcessAllSections);
     // LOS is an owning-ptr. Keep a non-owning one so that we can set the handle
     // below.
     auto *LOPtr = LO.get();
@@ -295,9 +299,10 @@ public:
   /// indirectly) will result in undefined behavior. If dependence tracking is
   /// required to detect or resolve such issues it should be added at a higher
   /// layer.
-  void removeObject(ObjHandleT H) {
+  Error removeObject(ObjHandleT H) {
     // How do we invalidate the symbols in H?
     LinkedObjList.erase(H);
+    return Error::success();
   }
 
   /// @brief Search for the given named symbol.
@@ -334,13 +339,15 @@ public:
   /// @brief Immediately emit and finalize the object set represented by the
   ///        given handle.
   /// @param H Handle for object set to emit/finalize.
-  void emitAndFinalize(ObjHandleT H) {
+  Error emitAndFinalize(ObjHandleT H) {
     (*H)->finalize();
+    return Error::success();
   }
 
 private:
 
   LinkedObjectListT LinkedObjList;
+  MemoryManagerGetter GetMemMgr;
   NotifyLoadedFtor NotifyLoaded;
   NotifyFinalizedFtor NotifyFinalized;
   bool ProcessAllSections = false;
diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h
index 1925489f79528..56aa04ce694a6 100644
--- a/include/llvm/ExecutionEngine/RuntimeDyld.h
+++ b/include/llvm/ExecutionEngine/RuntimeDyld.h
@@ -88,21 +88,6 @@ public:
     ObjSectionToIDMap ObjSecToIDMap;
   };
 
-  template <typename Derived> struct LoadedObjectInfoHelper : LoadedObjectInfo {
-  protected:
-    LoadedObjectInfoHelper(const LoadedObjectInfoHelper &) = default;
-    LoadedObjectInfoHelper() = default;
-
-  public:
-    LoadedObjectInfoHelper(RuntimeDyldImpl &RTDyld,
-                           LoadedObjectInfo::ObjSectionToIDMap ObjSecToIDMap)
-        : LoadedObjectInfo(RTDyld, std::move(ObjSecToIDMap)) {}
-
-    std::unique_ptr<llvm::LoadedObjectInfo> clone() const override {
-      return llvm::make_unique<Derived>(static_cast<const Derived &>(*this));
-    }
-  };
-
   /// \brief Memory Management.
   class MemoryManager {
     friend class RuntimeDyld;
diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h
index 8b3a90fa065b1..2e72c41ccee38 100644
--- a/include/llvm/IR/Constants.h
+++ b/include/llvm/IR/Constants.h
@@ -680,11 +680,6 @@ class ConstantDataArray final : public ConstantDataSequential {
   explicit ConstantDataArray(Type *ty, const char *Data)
       : ConstantDataSequential(ty, ConstantDataArrayVal, Data) {}
 
-  /// Allocate space for exactly zero operands.
-  void *operator new(size_t s) {
-    return User::operator new(s, 0);
-  }
-
 public:
   ConstantDataArray(const ConstantDataArray &) = delete;
 
@@ -739,11 +734,6 @@ class ConstantDataVector final : public ConstantDataSequential {
   explicit ConstantDataVector(Type *ty, const char *Data)
       : ConstantDataSequential(ty, ConstantDataVectorVal, Data) {}
 
-  // allocate space for exactly zero operands.
-  void *operator new(size_t s) {
-    return User::operator new(s, 0);
-  }
-
 public:
   ConstantDataVector(const ConstantDataVector &) = delete;
 
diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h
index ec33f82f70224..5344a93efb335 100644
--- a/include/llvm/IR/IRBuilder.h
+++ b/include/llvm/IR/IRBuilder.h
@@ -1062,7 +1062,7 @@ public:
 
   Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") {
     if (Constant *RC = dyn_cast<Constant>(RHS)) {
-      if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isAllOnesValue())
+      if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne())
         return LHS;  // LHS & -1 -> LHS
       if (Constant *LC = dyn_cast<Constant>(LHS))
         return Insert(Folder.CreateAnd(LC, RC), Name);
@@ -1203,22 +1203,22 @@ public:
     return SI;
   }
   FenceInst *CreateFence(AtomicOrdering Ordering,
-                         SynchronizationScope SynchScope = CrossThread,
+                         SyncScope::ID SSID = SyncScope::System,
                          const Twine &Name = "") {
-    return Insert(new FenceInst(Context, Ordering, SynchScope), Name);
+    return Insert(new FenceInst(Context, Ordering, SSID), Name);
   }
   AtomicCmpXchgInst *
   CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
                       AtomicOrdering SuccessOrdering,
                       AtomicOrdering FailureOrdering,
-                      SynchronizationScope SynchScope = CrossThread) {
+                      SyncScope::ID SSID = SyncScope::System) {
     return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering,
-                                        FailureOrdering, SynchScope));
+                                        FailureOrdering, SSID));
   }
   AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
                                  AtomicOrdering Ordering,
-                               SynchronizationScope SynchScope = CrossThread) {
-    return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SynchScope));
+                                 SyncScope::ID SSID = SyncScope::System) {
+    return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SSID));
   }
   Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
                    const Twine &Name = "") {
@@ -1517,11 +1517,9 @@ public:
                                 const Twine &Name = "") {
     if (V->getType() == DestTy)
       return V;
-    if (V->getType()->getScalarType()->isPointerTy() &&
-        DestTy->getScalarType()->isIntegerTy())
+    if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy())
       return CreatePtrToInt(V, DestTy, Name);
-    if (V->getType()->getScalarType()->isIntegerTy() &&
-        DestTy->getScalarType()->isPointerTy())
+    if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy())
       return CreateIntToPtr(V, DestTy, Name);
 
     return CreateBitCast(V, DestTy, Name);
diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h
index dc5f37450b480..60ae98869e552 100644
--- a/include/llvm/IR/Instructions.h
+++ b/include/llvm/IR/Instructions.h
@@ -52,11 +52,6 @@ class ConstantInt;
 class DataLayout;
 class LLVMContext;
 
-enum SynchronizationScope {
-  SingleThread = 0,
-  CrossThread = 1
-};
-
 //===----------------------------------------------------------------------===//
 //                                AllocaInst Class
 //===----------------------------------------------------------------------===//
@@ -195,17 +190,16 @@ public:
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, BasicBlock *InsertAtEnd);
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
-           AtomicOrdering Order, SynchronizationScope SynchScope = CrossThread,
+           AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
            Instruction *InsertBefore = nullptr)
       : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
-                 NameStr, isVolatile, Align, Order, SynchScope, InsertBefore) {}
+                 NameStr, isVolatile, Align, Order, SSID, InsertBefore) {}
   LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, AtomicOrdering Order,
-           SynchronizationScope SynchScope = CrossThread,
+           SyncScope::ID SSID = SyncScope::System,
            Instruction *InsertBefore = nullptr);
   LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
-           unsigned Align, AtomicOrdering Order,
-           SynchronizationScope SynchScope,
+           unsigned Align, AtomicOrdering Order, SyncScope::ID SSID,
            BasicBlock *InsertAtEnd);
   LoadInst(Value *Ptr, const char *NameStr, Instruction *InsertBefore);
   LoadInst(Value *Ptr, const char *NameStr, BasicBlock *InsertAtEnd);
@@ -235,34 +229,34 @@ public:
 
   void setAlignment(unsigned Align);
 
-  /// Returns the ordering effect of this fence.
+  /// Returns the ordering constraint of this load instruction.
   AtomicOrdering getOrdering() const {
     return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
   }
 
-  /// Set the ordering constraint on this load. May not be Release or
-  /// AcquireRelease.
+  /// Sets the ordering constraint of this load instruction.  May not be Release
+  /// or AcquireRelease.
   void setOrdering(AtomicOrdering Ordering) {
     setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
                                ((unsigned)Ordering << 7));
   }
 
-  SynchronizationScope getSynchScope() const {
-    return SynchronizationScope((getSubclassDataFromInstruction() >> 6) & 1);
+  /// Returns the synchronization scope ID of this load instruction.
+  SyncScope::ID getSyncScopeID() const {
+    return SSID;
   }
 
-  /// Specify whether this load is ordered with respect to all
-  /// concurrently executing threads, or only with respect to signal handlers
-  /// executing in the same thread.
-  void setSynchScope(SynchronizationScope xthread) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(1 << 6)) |
-                               (xthread << 6));
+  /// Sets the synchronization scope ID of this load instruction.
+  void setSyncScopeID(SyncScope::ID SSID) {
+    this->SSID = SSID;
   }
 
+  /// Sets the ordering constraint and the synchronization scope ID of this load
+  /// instruction.
   void setAtomic(AtomicOrdering Ordering,
-                 SynchronizationScope SynchScope = CrossThread) {
+                 SyncScope::ID SSID = SyncScope::System) {
     setOrdering(Ordering);
-    setSynchScope(SynchScope);
+    setSyncScopeID(SSID);
   }
 
   bool isSimple() const { return !isAtomic() && !isVolatile(); }
@@ -297,6 +291,11 @@ private:
   void setInstructionSubclassData(unsigned short D) {
     Instruction::setInstructionSubclassData(D);
   }
+
+  /// The synchronization scope ID of this load instruction.  Not quite enough
+  /// room in SubClassData for everything, so synchronization scope ID gets its
+  /// own field.
+  SyncScope::ID SSID;
 };
 
 //===----------------------------------------------------------------------===//
@@ -325,11 +324,10 @@ public:
             unsigned Align, BasicBlock *InsertAtEnd);
   StoreInst(Value *Val, Value *Ptr, bool isVolatile,
             unsigned Align, AtomicOrdering Order,
-            SynchronizationScope SynchScope = CrossThread,
+            SyncScope::ID SSID = SyncScope::System,
             Instruction *InsertBefore = nullptr);
   StoreInst(Value *Val, Value *Ptr, bool isVolatile,
-            unsigned Align, AtomicOrdering Order,
-            SynchronizationScope SynchScope,
+            unsigned Align, AtomicOrdering Order, SyncScope::ID SSID,
             BasicBlock *InsertAtEnd);
 
   // allocate space for exactly two operands
@@ -356,34 +354,34 @@ public:
 
   void setAlignment(unsigned Align);
 
-  /// Returns the ordering effect of this store.
+  /// Returns the ordering constraint of this store instruction.
   AtomicOrdering getOrdering() const {
     return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
   }
 
-  /// Set the ordering constraint on this store.  May not be Acquire or
-  /// AcquireRelease.
+  /// Sets the ordering constraint of this store instruction.  May not be
+  /// Acquire or AcquireRelease.
   void setOrdering(AtomicOrdering Ordering) {
     setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
                                ((unsigned)Ordering << 7));
   }
 
-  SynchronizationScope getSynchScope() const {
-    return SynchronizationScope((getSubclassDataFromInstruction() >> 6) & 1);
+  /// Returns the synchronization scope ID of this store instruction.
+  SyncScope::ID getSyncScopeID() const {
+    return SSID;
   }
 
-  /// Specify whether this store instruction is ordered with respect to all
-  /// concurrently executing threads, or only with respect to signal handlers
-  /// executing in the same thread.
-  void setSynchScope(SynchronizationScope xthread) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~(1 << 6)) |
-                               (xthread << 6));
+  /// Sets the synchronization scope ID of this store instruction.
+  void setSyncScopeID(SyncScope::ID SSID) {
+    this->SSID = SSID;
   }
 
+  /// Sets the ordering constraint and the synchronization scope ID of this
+  /// store instruction.
   void setAtomic(AtomicOrdering Ordering,
-                 SynchronizationScope SynchScope = CrossThread) {
+                 SyncScope::ID SSID = SyncScope::System) {
     setOrdering(Ordering);
-    setSynchScope(SynchScope);
+    setSyncScopeID(SSID);
   }
 
   bool isSimple() const { return !isAtomic() && !isVolatile(); }
@@ -421,6 +419,11 @@ private:
   void setInstructionSubclassData(unsigned short D) {
     Instruction::setInstructionSubclassData(D);
   }
+
+  /// The synchronization scope ID of this store instruction.  Not quite enough
+  /// room in SubClassData for everything, so synchronization scope ID gets its
+  /// own field.
+  SyncScope::ID SSID;
 };
 
 template <>
@@ -435,7 +438,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)
 
 /// An instruction for ordering other memory operations.
 class FenceInst : public Instruction {
-  void Init(AtomicOrdering Ordering, SynchronizationScope SynchScope);
+  void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
@@ -447,10 +450,9 @@ public:
   // Ordering may only be Acquire, Release, AcquireRelease, or
   // SequentiallyConsistent.
   FenceInst(LLVMContext &C, AtomicOrdering Ordering,
-            SynchronizationScope SynchScope = CrossThread,
+            SyncScope::ID SSID = SyncScope::System,
             Instruction *InsertBefore = nullptr);
-  FenceInst(LLVMContext &C, AtomicOrdering Ordering,
-            SynchronizationScope SynchScope,
+  FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
             BasicBlock *InsertAtEnd);
 
   // allocate space for exactly zero operands
@@ -458,28 +460,26 @@ public:
     return User::operator new(s, 0);
   }
 
-  /// Returns the ordering effect of this fence.
+  /// Returns the ordering constraint of this fence instruction.
   AtomicOrdering getOrdering() const {
     return AtomicOrdering(getSubclassDataFromInstruction() >> 1);
   }
 
-  /// Set the ordering constraint on this fence.  May only be Acquire, Release,
-  /// AcquireRelease, or SequentiallyConsistent.
+  /// Sets the ordering constraint of this fence instruction.  May only be
+  /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
   void setOrdering(AtomicOrdering Ordering) {
     setInstructionSubclassData((getSubclassDataFromInstruction() & 1) |
                                ((unsigned)Ordering << 1));
   }
 
-  SynchronizationScope getSynchScope() const {
-    return SynchronizationScope(getSubclassDataFromInstruction() & 1);
+  /// Returns the synchronization scope ID of this fence instruction.
+  SyncScope::ID getSyncScopeID() const {
+    return SSID;
   }
 
-  /// Specify whether this fence orders other operations with respect to all
-  /// concurrently executing threads, or only with respect to signal handlers
-  /// executing in the same thread.
-  void setSynchScope(SynchronizationScope xthread) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
-                               xthread);
+  /// Sets the synchronization scope ID of this fence instruction.
+  void setSyncScopeID(SyncScope::ID SSID) {
+    this->SSID = SSID;
   }
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -496,6 +496,11 @@ private:
   void setInstructionSubclassData(unsigned short D) {
     Instruction::setInstructionSubclassData(D);
   }
+
+  /// The synchronization scope ID of this fence instruction.  Not quite enough
+  /// room in SubClassData for everything, so synchronization scope ID gets its
+  /// own field.
+  SyncScope::ID SSID;
 };
 
 //===----------------------------------------------------------------------===//
@@ -509,7 +514,7 @@ private:
 class AtomicCmpXchgInst : public Instruction {
   void Init(Value *Ptr, Value *Cmp, Value *NewVal,
             AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
-            SynchronizationScope SynchScope);
+            SyncScope::ID SSID);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
@@ -521,13 +526,11 @@ public:
   AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
                     AtomicOrdering SuccessOrdering,
                     AtomicOrdering FailureOrdering,
-                    SynchronizationScope SynchScope,
-                    Instruction *InsertBefore = nullptr);
+                    SyncScope::ID SSID, Instruction *InsertBefore = nullptr);
   AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
                     AtomicOrdering SuccessOrdering,
                     AtomicOrdering FailureOrdering,
-                    SynchronizationScope SynchScope,
-                    BasicBlock *InsertAtEnd);
+                    SyncScope::ID SSID, BasicBlock *InsertAtEnd);
 
   // allocate space for exactly three operands
   void *operator new(size_t s) {
@@ -561,7 +564,12 @@ public:
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
-  /// Set the ordering constraint on this cmpxchg.
+  /// Returns the success ordering constraint of this cmpxchg instruction.
+  AtomicOrdering getSuccessOrdering() const {
+    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
+  }
+
+  /// Sets the success ordering constraint of this cmpxchg instruction.
   void setSuccessOrdering(AtomicOrdering Ordering) {
     assert(Ordering != AtomicOrdering::NotAtomic &&
            "CmpXchg instructions can only be atomic.");
@@ -569,6 +577,12 @@ public:
                                ((unsigned)Ordering << 2));
   }
 
+  /// Returns the failure ordering constraint of this cmpxchg instruction.
+  AtomicOrdering getFailureOrdering() const {
+    return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7);
+  }
+
+  /// Sets the failure ordering constraint of this cmpxchg instruction.
   void setFailureOrdering(AtomicOrdering Ordering) {
     assert(Ordering != AtomicOrdering::NotAtomic &&
            "CmpXchg instructions can only be atomic.");
@@ -576,28 +590,14 @@ public:
                                ((unsigned)Ordering << 5));
   }
 
-  /// Specify whether this cmpxchg is atomic and orders other operations with
-  /// respect to all concurrently executing threads, or only with respect to
-  /// signal handlers executing in the same thread.
-  void setSynchScope(SynchronizationScope SynchScope) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~2) |
-                               (SynchScope << 1));
-  }
-
-  /// Returns the ordering constraint on this cmpxchg.
-  AtomicOrdering getSuccessOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
-  }
-
-  /// Returns the ordering constraint on this cmpxchg.
-  AtomicOrdering getFailureOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7);
+  /// Returns the synchronization scope ID of this cmpxchg instruction.
+  SyncScope::ID getSyncScopeID() const {
+    return SSID;
   }
 
-  /// Returns whether this cmpxchg is atomic between threads or only within a
-  /// single thread.
-  SynchronizationScope getSynchScope() const {
-    return SynchronizationScope((getSubclassDataFromInstruction() & 2) >> 1);
+  /// Sets the synchronization scope ID of this cmpxchg instruction.
+  void setSyncScopeID(SyncScope::ID SSID) {
+    this->SSID = SSID;
   }
 
   Value *getPointerOperand() { return getOperand(0); }
@@ -652,6 +652,11 @@ private:
   void setInstructionSubclassData(unsigned short D) {
     Instruction::setInstructionSubclassData(D);
   }
+
+  /// The synchronization scope ID of this cmpxchg instruction.  Not quite
+  /// enough room in SubClassData for everything, so synchronization scope ID
+  /// gets its own field.
+  SyncScope::ID SSID;
 };
 
 template <>
@@ -711,10 +716,10 @@ public:
   };
 
   AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
-                AtomicOrdering Ordering, SynchronizationScope SynchScope,
+                AtomicOrdering Ordering, SyncScope::ID SSID,
                 Instruction *InsertBefore = nullptr);
   AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
-                AtomicOrdering Ordering, SynchronizationScope SynchScope,
+                AtomicOrdering Ordering, SyncScope::ID SSID,
                 BasicBlock *InsertAtEnd);
 
   // allocate space for exactly two operands
@@ -748,7 +753,12 @@ public:
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
-  /// Set the ordering constraint on this RMW.
+  /// Returns the ordering constraint of this rmw instruction.
+  AtomicOrdering getOrdering() const {
+    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
+  }
+
+  /// Sets the ordering constraint of this rmw instruction.
   void setOrdering(AtomicOrdering Ordering) {
     assert(Ordering != AtomicOrdering::NotAtomic &&
            "atomicrmw instructions can only be atomic.");
@@ -756,23 +766,14 @@ public:
                                ((unsigned)Ordering << 2));
   }
 
-  /// Specify whether this RMW orders other operations with respect to all
-  /// concurrently executing threads, or only with respect to signal handlers
-  /// executing in the same thread.
-  void setSynchScope(SynchronizationScope SynchScope) {
-    setInstructionSubclassData((getSubclassDataFromInstruction() & ~2) |
-                               (SynchScope << 1));
+  /// Returns the synchronization scope ID of this rmw instruction.
+  SyncScope::ID getSyncScopeID() const {
+    return SSID;
   }
 
-  /// Returns the ordering constraint on this RMW.
-  AtomicOrdering getOrdering() const {
-    return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
-  }
-
-  /// Returns whether this RMW is atomic between threads or only within a
-  /// single thread.
-  SynchronizationScope getSynchScope() const {
-    return SynchronizationScope((getSubclassDataFromInstruction() & 2) >> 1);
+  /// Sets the synchronization scope ID of this rmw instruction.
+  void setSyncScopeID(SyncScope::ID SSID) {
+    this->SSID = SSID;
   }
 
   Value *getPointerOperand() { return getOperand(0); }
@@ -797,13 +798,18 @@ public:
 
 private:
   void Init(BinOp Operation, Value *Ptr, Value *Val,
-            AtomicOrdering Ordering, SynchronizationScope SynchScope);
+            AtomicOrdering Ordering, SyncScope::ID SSID);
 
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
   void setInstructionSubclassData(unsigned short D) {
     Instruction::setInstructionSubclassData(D);
   }
+
+  /// The synchronization scope ID of this rmw instruction.  Not quite enough
+  /// room in SubClassData for everything, so synchronization scope ID gets its
+  /// own field.
+  SyncScope::ID SSID;
 };
 
 template <>
@@ -1101,8 +1107,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)
 /// Represent an integer comparison operator.
 class ICmpInst: public CmpInst {
   void AssertOK() {
-    assert(getPredicate() >= CmpInst::FIRST_ICMP_PREDICATE &&
-           getPredicate() <= CmpInst::LAST_ICMP_PREDICATE &&
+    assert(isIntPredicate() &&
            "Invalid ICmp predicate value");
     assert(getOperand(0)->getType() == getOperand(1)->getType() &&
           "Both operands to ICmp instruction are not of the same type!");
@@ -1244,8 +1249,7 @@ public:
 /// Represents a floating point comparison operator.
 class FCmpInst: public CmpInst {
   void AssertOK() {
-    assert(getPredicate() <= FCmpInst::LAST_FCMP_PREDICATE &&
-           "Invalid FCmp predicate value");
+    assert(isFPPredicate() && "Invalid FCmp predicate value");
     assert(getOperand(0)->getType() == getOperand(1)->getType() &&
            "Both operands to FCmp instruction are not of the same type!");
     // Check that the operands are the right type
diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h
index 944af57a7800c..f55d17ec72c8a 100644
--- a/include/llvm/IR/IntrinsicInst.h
+++ b/include/llvm/IR/IntrinsicInst.h
@@ -296,6 +296,175 @@ namespace llvm {
     }
   };
 
+  class ElementUnorderedAtomicMemMoveInst : public IntrinsicInst {
+  private:
+    enum { ARG_DEST = 0, ARG_SOURCE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 };
+
+  public:
+    Value *getRawDest() const {
+      return const_cast<Value *>(getArgOperand(ARG_DEST));
+    }
+    const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); }
+    Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); }
+
+    /// Return the arguments to the instruction.
+    Value *getRawSource() const {
+      return const_cast<Value *>(getArgOperand(ARG_SOURCE));
+    }
+    const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); }
+    Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); }
+
+    Value *getLength() const {
+      return const_cast<Value *>(getArgOperand(ARG_LENGTH));
+    }
+    const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); }
+    Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); }
+
+    bool isVolatile() const { return false; }
+
+    Value *getRawElementSizeInBytes() const {
+      return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE));
+    }
+
+    ConstantInt *getElementSizeInBytesCst() const {
+      return cast<ConstantInt>(getRawElementSizeInBytes());
+    }
+
+    uint32_t getElementSizeInBytes() const {
+      return getElementSizeInBytesCst()->getZExtValue();
+    }
+
+    /// This is just like getRawDest, but it strips off any cast
+    /// instructions that feed it, giving the original input.  The returned
+    /// value is guaranteed to be a pointer.
+    Value *getDest() const { return getRawDest()->stripPointerCasts(); }
+
+    /// This is just like getRawSource, but it strips off any cast
+    /// instructions that feed it, giving the original input.  The returned
+    /// value is guaranteed to be a pointer.
+    Value *getSource() const { return getRawSource()->stripPointerCasts(); }
+
+    unsigned getDestAddressSpace() const {
+      return cast<PointerType>(getRawDest()->getType())->getAddressSpace();
+    }
+
+    unsigned getSourceAddressSpace() const {
+      return cast<PointerType>(getRawSource()->getType())->getAddressSpace();
+    }
+
+    /// Set the specified arguments of the instruction.
+    void setDest(Value *Ptr) {
+      assert(getRawDest()->getType() == Ptr->getType() &&
+             "setDest called with pointer of wrong type!");
+      setArgOperand(ARG_DEST, Ptr);
+    }
+
+    void setSource(Value *Ptr) {
+      assert(getRawSource()->getType() == Ptr->getType() &&
+             "setSource called with pointer of wrong type!");
+      setArgOperand(ARG_SOURCE, Ptr);
+    }
+
+    void setLength(Value *L) {
+      assert(getLength()->getType() == L->getType() &&
+             "setLength called with value of wrong type!");
+      setArgOperand(ARG_LENGTH, L);
+    }
+
+    void setElementSizeInBytes(Constant *V) {
+      assert(V->getType() == Type::getInt8Ty(getContext()) &&
+             "setElementSizeInBytes called with value of wrong type!");
+      setArgOperand(ARG_ELEMENTSIZE, V);
+    }
+
+    static inline bool classof(const IntrinsicInst *I) {
+      return I->getIntrinsicID() == Intrinsic::memmove_element_unordered_atomic;
+    }
+    static inline bool classof(const Value *V) {
+      return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
+    }
+  };
+
+  /// This class represents atomic memset intrinsic
+  /// TODO: Integrate this class into MemIntrinsic hierarchy; for now this is
+  /// C&P of all methods from that hierarchy
+  class ElementUnorderedAtomicMemSetInst : public IntrinsicInst {
+  private:
+    enum { ARG_DEST = 0, ARG_VALUE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 };
+
+  public:
+    Value *getRawDest() const {
+      return const_cast<Value *>(getArgOperand(ARG_DEST));
+    }
+    const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); }
+    Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); }
+
+    Value *getValue() const { return const_cast<Value*>(getArgOperand(ARG_VALUE)); }
+    const Use &getValueUse() const { return getArgOperandUse(ARG_VALUE); }
+    Use &getValueUse() { return getArgOperandUse(ARG_VALUE); }
+
+    Value *getLength() const {
+      return const_cast<Value *>(getArgOperand(ARG_LENGTH));
+    }
+    const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); }
+    Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); }
+
+    bool isVolatile() const { return false; }
+
+    Value *getRawElementSizeInBytes() const {
+      return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE));
+    }
+
+    ConstantInt *getElementSizeInBytesCst() const {
+      return cast<ConstantInt>(getRawElementSizeInBytes());
+    }
+
+    uint32_t getElementSizeInBytes() const {
+      return getElementSizeInBytesCst()->getZExtValue();
+    }
+
+    /// This is just like getRawDest, but it strips off any cast
+    /// instructions that feed it, giving the original input.  The returned
+    /// value is guaranteed to be a pointer.
+    Value *getDest() const { return getRawDest()->stripPointerCasts(); }
+
+    unsigned getDestAddressSpace() const {
+      return cast<PointerType>(getRawDest()->getType())->getAddressSpace();
+    }
+
+    /// Set the specified arguments of the instruction.
+    void setDest(Value *Ptr) {
+      assert(getRawDest()->getType() == Ptr->getType() &&
+             "setDest called with pointer of wrong type!");
+      setArgOperand(ARG_DEST, Ptr);
+    }
+
+    void setValue(Value *Val) {
+      assert(getValue()->getType() == Val->getType() &&
+             "setValue called with value of wrong type!");
+      setArgOperand(ARG_VALUE, Val);
+    }
+
+    void setLength(Value *L) {
+      assert(getLength()->getType() == L->getType() &&
+             "setLength called with value of wrong type!");
+      setArgOperand(ARG_LENGTH, L);
+    }
+
+    void setElementSizeInBytes(Constant *V) {
+      assert(V->getType() == Type::getInt8Ty(getContext()) &&
+             "setElementSizeInBytes called with value of wrong type!");
+      setArgOperand(ARG_ELEMENTSIZE, V);
+    }
+
+    static inline bool classof(const IntrinsicInst *I) {
+      return I->getIntrinsicID() == Intrinsic::memset_element_unordered_atomic;
+    }
+    static inline bool classof(const Value *V) {
+      return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
+    }
+  };
+
   /// This is the common base class for memset/memcpy/memmove.
   class MemIntrinsic : public IntrinsicInst {
   public:
diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td
index 45936a6e9b66d..14c88e5194356 100644
--- a/include/llvm/IR/Intrinsics.td
+++ b/include/llvm/IR/Intrinsics.td
@@ -873,6 +873,22 @@ def int_memcpy_element_unordered_atomic
                   ReadOnly<1>
                 ]>;
 
+// @llvm.memmove.element.unordered.atomic.*(dest, src, length, elementsize)
+def int_memmove_element_unordered_atomic
+    : Intrinsic<[],
+                [
+                  llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty
+                ],
+                [
+                  IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>,
+                  ReadOnly<1>
+                ]>;
+
+// @llvm.memset.element.unordered.atomic.*(dest, value, length, elementsize)
+def int_memset_element_unordered_atomic
+    : Intrinsic<[], [ llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i32_ty ],
+                [ IntrArgMemOnly, NoCapture<0>, WriteOnly<0> ]>;
+
 //===------------------------ Reduction Intrinsics ------------------------===//
 //
 def int_experimental_vector_reduce_fadd : Intrinsic<[llvm_anyfloat_ty],
diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h
index b27abad618c99..4cb77701f7620 100644
--- a/include/llvm/IR/LLVMContext.h
+++ b/include/llvm/IR/LLVMContext.h
@@ -42,6 +42,24 @@ class Output;
 
 } // end namespace yaml
 
+namespace SyncScope {
+
+typedef uint8_t ID;
+
+/// Known synchronization scope IDs, which always have the same value.  All
+/// synchronization scope IDs that LLVM has special knowledge of are listed
+/// here.  Additionally, this scheme allows LLVM to efficiently check for
+/// specific synchronization scope ID without comparing strings.
+enum {
+  /// Synchronized with respect to signal handlers executing in the same thread.
+  SingleThread = 0,
+
+  /// Synchronized with respect to all concurrently executing threads.
+  System = 1
+};
+
+} // end namespace SyncScope
+
 /// This is an important class for using LLVM in a threaded context.  It
 /// (opaquely) owns and manages the core "global" data of LLVM's core
 /// infrastructure, including the type and constant uniquing tables.
@@ -111,6 +129,16 @@ public:
   /// tag registered with an LLVMContext has an unique ID.
   uint32_t getOperandBundleTagID(StringRef Tag) const;
 
+  /// getOrInsertSyncScopeID - Maps synchronization scope name to
+  /// synchronization scope ID.  Every synchronization scope registered with
+  /// LLVMContext has unique ID except pre-defined ones.
+  SyncScope::ID getOrInsertSyncScopeID(StringRef SSN);
+
+  /// getSyncScopeNames - Populates client supplied SmallVector with
+  /// synchronization scope names registered with LLVMContext.  Synchronization
+  /// scope names are ordered by increasing synchronization scope IDs.
+  void getSyncScopeNames(SmallVectorImpl<StringRef> &SSNs) const;
+
   /// Define the GC for a function
   void setGC(const Function &Fn, std::string GCName);
 
diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h
index d47d82a57bffb..196e32e3615c0 100644
--- a/include/llvm/IR/Module.h
+++ b/include/llvm/IR/Module.h
@@ -249,7 +249,7 @@ public:
   /// when other randomness consuming passes are added or removed. In
   /// addition, the random stream will be reproducible across LLVM
   /// versions when the pass does not change.
-  RandomNumberGenerator *createRNG(const Pass* P) const;
+  std::unique_ptr<RandomNumberGenerator> createRNG(const Pass* P) const;
 
 /// @}
 /// @name Module Level Mutators
diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h
index b43d588658628..4aa8a0199ab15 100644
--- a/include/llvm/IR/ModuleSummaryIndex.h
+++ b/include/llvm/IR/ModuleSummaryIndex.h
@@ -47,7 +47,13 @@ template <typename T> struct MappingTraits;
 
 /// \brief Class to accumulate and hold information about a callee.
 struct CalleeInfo {
-  enum class HotnessType : uint8_t { Unknown = 0, Cold = 1, None = 2, Hot = 3 };
+  enum class HotnessType : uint8_t {
+    Unknown = 0,
+    Cold = 1,
+    None = 2,
+    Hot = 3,
+    Critical = 4
+  };
   HotnessType Hotness = HotnessType::Unknown;
 
   CalleeInfo() = default;
@@ -516,7 +522,7 @@ using ModulePathStringTableTy = StringMap<std::pair<uint64_t, ModuleHash>>;
 
 /// Map of global value GUID to its summary, used to identify values defined in
 /// a particular module, and provide efficient access to their summary.
-using GVSummaryMapTy = std::map<GlobalValue::GUID, GlobalValueSummary *>;
+using GVSummaryMapTy = DenseMap<GlobalValue::GUID, GlobalValueSummary *>;
 
 /// Class to hold module path string table and global value map,
 /// and encapsulate methods for operating on them.
diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h
index d03b7b65f81e2..393175675034d 100644
--- a/include/llvm/IR/PassManager.h
+++ b/include/llvm/IR/PassManager.h
@@ -162,6 +162,14 @@ public:
     return PA;
   }
 
+  /// \brief Construct a preserved analyses object with a single preserved set.
+  template <typename AnalysisSetT>
+  static PreservedAnalyses allInSet() {
+    PreservedAnalyses PA;
+    PA.preserveSet<AnalysisSetT>();
+    return PA;
+  }
+
   /// Mark an analysis as preserved.
   template <typename AnalysisT> void preserve() { preserve(AnalysisT::ID()); }
 
@@ -1062,10 +1070,27 @@ public:
 
     const AnalysisManagerT &getManager() const { return *AM; }
 
-    /// \brief Handle invalidation by ignoring it; this pass is immutable.
+    /// When invalidation occurs, remove any registered invalidation events.
     bool invalidate(
-        IRUnitT &, const PreservedAnalyses &,
-        typename AnalysisManager<IRUnitT, ExtraArgTs...>::Invalidator &) {
+        IRUnitT &IRUnit, const PreservedAnalyses &PA,
+        typename AnalysisManager<IRUnitT, ExtraArgTs...>::Invalidator &Inv) {
+      // Loop over the set of registered outer invalidation mappings and if any
+      // of them map to an analysis that is now invalid, clear it out.
+      SmallVector<AnalysisKey *, 4> DeadKeys;
+      for (auto &KeyValuePair : OuterAnalysisInvalidationMap) {
+        AnalysisKey *OuterID = KeyValuePair.first;
+        auto &InnerIDs = KeyValuePair.second;
+        InnerIDs.erase(llvm::remove_if(InnerIDs, [&](AnalysisKey *InnerID) {
+          return Inv.invalidate(InnerID, IRUnit, PA); }),
+                       InnerIDs.end());
+        if (InnerIDs.empty())
+          DeadKeys.push_back(OuterID);
+      }
+
+      for (auto OuterID : DeadKeys)
+        OuterAnalysisInvalidationMap.erase(OuterID);
+
+      // The proxy itself remains valid regardless of anything else.
       return false;
     }
 
diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h
index 5b69e7855cc77..acb8952116441 100644
--- a/include/llvm/IR/PatternMatch.h
+++ b/include/llvm/IR/PatternMatch.h
@@ -158,12 +158,18 @@ struct match_neg_zero {
 /// zero
 inline match_neg_zero m_NegZero() { return match_neg_zero(); }
 
+struct match_any_zero {
+  template <typename ITy> bool match(ITy *V) {
+    if (const auto *C = dyn_cast<Constant>(V))
+      return C->isZeroValue();
+    return false;
+  }
+};
+
 /// \brief - Match an arbitrary zero/null constant.  This includes
 /// zero_initializer for vectors and ConstantPointerNull for pointers. For
 /// floating point constants, this will match negative zero and positive zero
-inline match_combine_or<match_zero, match_neg_zero> m_AnyZero() {
-  return m_CombineOr(m_Zero(), m_NegZero());
-}
+inline match_any_zero m_AnyZero() { return match_any_zero(); }
 
 struct match_nan {
   template <typename ITy> bool match(ITy *V) {
@@ -176,6 +182,39 @@ struct match_nan {
 /// Match an arbitrary NaN constant. This includes quiet and signalling nans.
 inline match_nan m_NaN() { return match_nan(); }
 
+struct match_one {
+  template <typename ITy> bool match(ITy *V) {
+    if (const auto *C = dyn_cast<Constant>(V))
+      return C->isOneValue();
+    return false;
+  }
+};
+
+/// \brief Match an integer 1 or a vector with all elements equal to 1.
+inline match_one m_One() { return match_one(); }
+
+struct match_all_ones {
+  template <typename ITy> bool match(ITy *V) {
+    if (const auto *C = dyn_cast<Constant>(V))
+      return C->isAllOnesValue();
+    return false;
+  }
+};
+
+/// \brief Match an integer or vector with all bits set to true.
+inline match_all_ones m_AllOnes() { return match_all_ones(); }
+
+struct match_sign_mask {
+  template <typename ITy> bool match(ITy *V) {
+    if (const auto *C = dyn_cast<Constant>(V))
+      return C->isMinSignedValue();
+    return false;
+  }
+};
+
+/// \brief Match an integer or vector with only the sign bit(s) set.
+inline match_sign_mask m_SignMask() { return match_sign_mask(); }
+
 struct apint_match {
   const APInt *&Res;
 
@@ -259,34 +298,6 @@ template <typename Predicate> struct api_pred_ty : public Predicate {
   }
 };
 
-struct is_one {
-  bool isValue(const APInt &C) { return C.isOneValue(); }
-};
-
-/// \brief Match an integer 1 or a vector with all elements equal to 1.
-inline cst_pred_ty<is_one> m_One() { return cst_pred_ty<is_one>(); }
-inline api_pred_ty<is_one> m_One(const APInt *&V) { return V; }
-
-struct is_all_ones {
-  bool isValue(const APInt &C) { return C.isAllOnesValue(); }
-};
-
-/// \brief Match an integer or vector with all bits set to true.
-inline cst_pred_ty<is_all_ones> m_AllOnes() {
-  return cst_pred_ty<is_all_ones>();
-}
-inline api_pred_ty<is_all_ones> m_AllOnes(const APInt *&V) { return V; }
-
-struct is_sign_mask {
-  bool isValue(const APInt &C) { return C.isSignMask(); }
-};
-
-/// \brief Match an integer or vector with only the sign bit(s) set.
-inline cst_pred_ty<is_sign_mask> m_SignMask() {
-  return cst_pred_ty<is_sign_mask>();
-}
-inline api_pred_ty<is_sign_mask> m_SignMask(const APInt *&V) { return V; }
-
 struct is_power2 {
   bool isValue(const APInt &C) { return C.isPowerOf2(); }
 };
diff --git a/include/llvm/IR/SafepointIRVerifier.h b/include/llvm/IR/SafepointIRVerifier.h
new file mode 100644
index 0000000000000..092050d1d2079
--- /dev/null
+++ b/include/llvm/IR/SafepointIRVerifier.h
@@ -0,0 +1,35 @@
+//===- SafepointIRVerifier.h - Checks for GC relocation problems *- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a verifier which is useful for enforcing the relocation
+// properties required by a relocating GC.  Specifically, it looks for uses of
+// the unrelocated value of pointer SSA values after a possible safepoint. It
+// attempts to report no false negatives, but may end up reporting false
+// positives in rare cases (see the note at the top of the corresponding cpp
+// file.)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_SAFEPOINT_IR_VERIFIER
+#define LLVM_IR_SAFEPOINT_IR_VERIFIER
+
+namespace llvm {
+
+class Function;
+class FunctionPass;
+
+/// Run the safepoint verifier over a single function.  Crashes on failure.
+void verifySafepointIR(Function &F);
+
+/// Create an instance of the safepoint verifier pass which can be added to
+/// a pass pipeline to check for relocation bugs.
+FunctionPass *createSafepointIRVerifierPass();
+}
+
+#endif // LLVM_IR_SAFEPOINT_IR_VERIFIER
diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h
index b37b59288e3f4..ef7801266777c 100644
--- a/include/llvm/IR/Type.h
+++ b/include/llvm/IR/Type.h
@@ -202,6 +202,12 @@ public:
   /// Return true if this is an integer type or a vector of integer types.
   bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }
 
+  /// Return true if this is an integer type or a vector of integer types of
+  /// the given width.
+  bool isIntOrIntVectorTy(unsigned BitWidth) const {
+    return getScalarType()->isIntegerTy(BitWidth);
+  }
+
   /// True if this is an instance of FunctionType.
   bool isFunctionTy() const { return getTypeID() == FunctionTyID; }
 
diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h
index aab14070dbdac..39ac4649b70d4 100644
--- a/include/llvm/InitializePasses.h
+++ b/include/llvm/InitializePasses.h
@@ -303,6 +303,7 @@ void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&);
 void initializePromoteLegacyPassPass(PassRegistry&);
 void initializePruneEHPass(PassRegistry&);
 void initializeRABasicPass(PassRegistry&);
+void initializeRAFastPass(PassRegistry&);
 void initializeRAGreedyPass(PassRegistry&);
 void initializeReassociateLegacyPassPass(PassRegistry&);
 void initializeRegBankSelectPass(PassRegistry&);
@@ -318,6 +319,7 @@ void initializeResetMachineFunctionPass(PassRegistry&);
 void initializeReversePostOrderFunctionAttrsLegacyPassPass(PassRegistry&);
 void initializeRewriteStatepointsForGCPass(PassRegistry&);
 void initializeRewriteSymbolsLegacyPassPass(PassRegistry&);
+void initializeSafepointIRVerifierPass(PassRegistry&);
 void initializeSCCPLegacyPassPass(PassRegistry&);
 void initializeSCEVAAWrapperPassPass(PassRegistry&);
 void initializeSLPVectorizerPass(PassRegistry&);
diff --git a/include/llvm/MC/MCAsmBackend.h b/include/llvm/MC/MCAsmBackend.h
index c9c43a22da5db..5a8e29d08ad23 100644
--- a/include/llvm/MC/MCAsmBackend.h
+++ b/include/llvm/MC/MCAsmBackend.h
@@ -73,7 +73,7 @@ public:
   /// reported via \p Ctx.
   virtual void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                           const MCValue &Target, MutableArrayRef<char> Data,
-                          uint64_t Value, bool IsPCRel) const = 0;
+                          uint64_t Value, bool IsResolved) const = 0;
 
   /// @}
 
diff --git a/include/llvm/MC/MCDisassembler/MCDisassembler.h b/include/llvm/MC/MCDisassembler/MCDisassembler.h
index 5e626f1869861..7f09c05ccf2ab 100644
--- a/include/llvm/MC/MCDisassembler/MCDisassembler.h
+++ b/include/llvm/MC/MCDisassembler/MCDisassembler.h
@@ -68,6 +68,7 @@ public:
   ///                   an invalid instruction.
   /// \param Address  - The address, in the memory space of region, of the first
   ///                   byte of the instruction.
+  /// \param Bytes    - A reference to the actual bytes of the instruction.
   /// \param VStream  - The stream to print warnings and diagnostic messages on.
   /// \param CStream  - The stream to print comments and annotations on.
   /// \return         - MCDisassembler::Success if the instruction is valid,
diff --git a/include/llvm/MC/MCMachObjectWriter.h b/include/llvm/MC/MCMachObjectWriter.h
index 2d2480a27223c..42dc90da3049a 100644
--- a/include/llvm/MC/MCMachObjectWriter.h
+++ b/include/llvm/MC/MCMachObjectWriter.h
@@ -233,8 +233,7 @@ public:
 
   void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                         const MCFragment *Fragment, const MCFixup &Fixup,
-                        MCValue Target, bool &IsPCRel,
-                        uint64_t &FixedValue) override;
+                        MCValue Target, uint64_t &FixedValue) override;
 
   void bindIndirectSymbols(MCAssembler &Asm);
 
diff --git a/include/llvm/MC/MCObjectWriter.h b/include/llvm/MC/MCObjectWriter.h
index 86bcbb6861d7f..cd90690fb1863 100644
--- a/include/llvm/MC/MCObjectWriter.h
+++ b/include/llvm/MC/MCObjectWriter.h
@@ -86,7 +86,7 @@ public:
   virtual void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                                 const MCFragment *Fragment,
                                 const MCFixup &Fixup, MCValue Target,
-                                bool &IsPCRel, uint64_t &FixedValue) = 0;
+                                uint64_t &FixedValue) = 0;
 
   /// Check whether the difference (A - B) between two symbol references is
   /// fully resolved.
diff --git a/include/llvm/MC/MCSymbolWasm.h b/include/llvm/MC/MCSymbolWasm.h
index 7ea89629efda6..9bae6c582faa3 100644
--- a/include/llvm/MC/MCSymbolWasm.h
+++ b/include/llvm/MC/MCSymbolWasm.h
@@ -21,6 +21,8 @@ private:
   std::string ModuleName;
   SmallVector<wasm::ValType, 1> Returns;
   SmallVector<wasm::ValType, 4> Params;
+  bool ParamsSet = false;
+  bool ReturnsSet = false;
 
   /// An expression describing how to calculate the size of a symbol. If a
   /// symbol has no size this field will be NULL.
@@ -45,15 +47,23 @@ public:
 
   const StringRef getModuleName() const { return ModuleName; }
 
-  const SmallVector<wasm::ValType, 1> &getReturns() const { return Returns; }
+  const SmallVector<wasm::ValType, 1> &getReturns() const {
+    assert(ReturnsSet);
+    return Returns;
+  }
 
   void setReturns(SmallVectorImpl<wasm::ValType> &&Rets) {
+    ReturnsSet = true;
     Returns = std::move(Rets);
   }
 
-  const SmallVector<wasm::ValType, 4> &getParams() const { return Params; }
+  const SmallVector<wasm::ValType, 4> &getParams() const {
+    assert(ParamsSet);
+    return Params;
+  }
 
   void setParams(SmallVectorImpl<wasm::ValType> &&Pars) {
+    ParamsSet = true;
     Params = std::move(Pars);
   }
 };
diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h
index 78e0b5f6ed306..89c1ba6be35f0 100644
--- a/include/llvm/Object/COFF.h
+++ b/include/llvm/Object/COFF.h
@@ -698,6 +698,9 @@ struct coff_resource_dir_entry {
     uint32_t getNameOffset() const {
       return maskTrailingOnes<uint32_t>(31) & NameOffset;
     }
+    // Even though the PE/COFF spec doesn't mention this, the high bit of a name
+    // offset is set.
+    void setNameOffset(uint32_t Offset) { NameOffset = Offset | (1 << 31); }
   } Identifier;
   union {
     support::ulittle32_t DataEntryOffset;
diff --git a/include/llvm/Object/Wasm.h b/include/llvm/Object/Wasm.h
index 5c8445f10f448..07ee4a4d6c4da 100644
--- a/include/llvm/Object/Wasm.h
+++ b/include/llvm/Object/Wasm.h
@@ -61,7 +61,7 @@ public:
 
   void print(raw_ostream &Out) const {
     Out << "Name=" << Name << ", Type=" << static_cast<int>(Type)
-        << ", Flags=" << Flags;
+        << ", Flags=" << Flags << " ElemIndex=" << ElementIndex;
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -69,8 +69,7 @@ public:
 #endif
 };
 
-class WasmSection {
-public:
+struct WasmSection {
   WasmSection() = default;
 
   uint32_t Type = 0; // Section type (See below)
@@ -80,6 +79,11 @@ public:
   std::vector<wasm::WasmRelocation> Relocations; // Relocations for this section
 };
 
+struct WasmSegment {
+  uint32_t SectionOffset;
+  wasm::WasmDataSegment Data;
+};
+
 class WasmObjectFile : public ObjectFile {
 
 public:
@@ -110,7 +114,7 @@ public:
     return ElemSegments;
   }
 
-  const std::vector<wasm::WasmDataSegment>& dataSegments() const {
+  const std::vector<WasmSegment>& dataSegments() const {
     return DataSegments;
   }
 
@@ -210,7 +214,7 @@ private:
   std::vector<wasm::WasmImport> Imports;
   std::vector<wasm::WasmExport> Exports;
   std::vector<wasm::WasmElemSegment> ElemSegments;
-  std::vector<wasm::WasmDataSegment> DataSegments;
+  std::vector<WasmSegment> DataSegments;
   std::vector<wasm::WasmFunction> Functions;
   std::vector<WasmSymbol> Symbols;
   ArrayRef<uint8_t> CodeSection;
diff --git a/include/llvm/ObjectYAML/WasmYAML.h b/include/llvm/ObjectYAML/WasmYAML.h
index 6bf08d340eeb1..709ad8ec3b776 100644
--- a/include/llvm/ObjectYAML/WasmYAML.h
+++ b/include/llvm/ObjectYAML/WasmYAML.h
@@ -98,7 +98,8 @@ struct Relocation {
 };
 
 struct DataSegment {
-  uint32_t Index;
+  uint32_t MemoryIndex;
+  uint32_t SectionOffset;
   wasm::WasmInitExpr Offset;
   yaml::BinaryRef Content;
 };
diff --git a/include/llvm/Option/OptTable.h b/include/llvm/Option/OptTable.h
index 3e7b019a0d4ea..a35e182f00e53 100644
--- a/include/llvm/Option/OptTable.h
+++ b/include/llvm/Option/OptTable.h
@@ -140,7 +140,8 @@ public:
   //  to start with.
   ///
   /// \return The vector of flags which start with Cur.
-  std::vector<std::string> findByPrefix(StringRef Cur) const;
+  std::vector<std::string> findByPrefix(StringRef Cur,
+                                        unsigned short DisableFlags) const;
 
   /// \brief Parse a single argument; returning the new argument and
   /// updating Index.
diff --git a/include/llvm/Passes/PassBuilder.h b/include/llvm/Passes/PassBuilder.h
index ff19583973318..33433f6b4a104 100644
--- a/include/llvm/Passes/PassBuilder.h
+++ b/include/llvm/Passes/PassBuilder.h
@@ -46,6 +46,19 @@ class PassBuilder {
   Optional<PGOOptions> PGOOpt;
 
 public:
+  /// \brief A struct to capture parsed pass pipeline names.
+  ///
+  /// A pipeline is defined as a series of names, each of which may in itself
+  /// recursively contain a nested pipeline. A name is either the name of a pass
+  /// (e.g. "instcombine") or the name of a pipeline type (e.g. "cgscc"). If the
+  /// name is the name of a pass, the InnerPipeline is empty, since passes
+  /// cannot contain inner pipelines. See parsePassPipeline() for a more
+  /// detailed description of the textual pipeline format.
+  struct PipelineElement {
+    StringRef Name;
+    std::vector<PipelineElement> InnerPipeline;
+  };
+
   /// \brief LLVM-provided high-level optimization levels.
   ///
   /// This enumerates the LLVM-provided high-level optimization levels. Each
@@ -188,9 +201,14 @@ public:
   /// only intended for use when attempting to optimize code. If frontends
   /// require some transformations for semantic reasons, they should explicitly
   /// build them.
+  ///
+  /// \p PrepareForThinLTO indicates whether this is invoked in
+  /// PrepareForThinLTO phase. Special handling is needed for sample PGO to
+  /// ensure profile accurate in the backend profile annotation phase.
   FunctionPassManager
   buildFunctionSimplificationPipeline(OptimizationLevel Level,
-                                      bool DebugLogging = false);
+                                      bool DebugLogging = false,
+                                      bool PrepareForThinLTO = false);
 
   /// Construct the core LLVM module canonicalization and simplification
   /// pipeline.
@@ -205,9 +223,14 @@ public:
   /// only intended for use when attempting to optimize code. If frontends
   /// require some transformations for semantic reasons, they should explicitly
   /// build them.
+  ///
+  /// \p PrepareForThinLTO indicates whether this is invoked in
+  /// PrepareForThinLTO phase. Special handling is needed for sample PGO to
+  /// ensure profile accurate in the backend profile annotation phase.
   ModulePassManager
   buildModuleSimplificationPipeline(OptimizationLevel Level,
-                                    bool DebugLogging = false);
+                                    bool DebugLogging = false,
+                                    bool PrepareForThinLTO = false);
 
   /// Construct the core LLVM module optimization pipeline.
   ///
@@ -302,7 +325,8 @@ public:
   /// registered.
   AAManager buildDefaultAAPipeline();
 
-  /// \brief Parse a textual pass pipeline description into a \c ModulePassManager.
+  /// \brief Parse a textual pass pipeline description into a \c
+  /// ModulePassManager.
   ///
   /// The format of the textual pass pipeline description looks something like:
   ///
@@ -312,8 +336,8 @@ public:
   /// are comma separated. As a special shortcut, if the very first pass is not
   /// a module pass (as a module pass manager is), this will automatically form
   /// the shortest stack of pass managers that allow inserting that first pass.
-  /// So, assuming function passes 'fpassN', CGSCC passes 'cgpassN', and loop passes
-  /// 'lpassN', all of these are valid:
+  /// So, assuming function passes 'fpassN', CGSCC passes 'cgpassN', and loop
+  /// passes 'lpassN', all of these are valid:
   ///
   ///   fpass1,fpass2,fpass3
   ///   cgpass1,cgpass2,cgpass3
@@ -326,13 +350,28 @@ public:
   ///   module(function(loop(lpass1,lpass2,lpass3)))
   ///
   /// This shortcut is especially useful for debugging and testing small pass
-  /// combinations. Note that these shortcuts don't introduce any other magic. If
-  /// the sequence of passes aren't all the exact same kind of pass, it will be
-  /// an error. You cannot mix different levels implicitly, you must explicitly
-  /// form a pass manager in which to nest passes.
+  /// combinations. Note that these shortcuts don't introduce any other magic.
+  /// If the sequence of passes aren't all the exact same kind of pass, it will
+  /// be an error. You cannot mix different levels implicitly, you must
+  /// explicitly form a pass manager in which to nest passes.
   bool parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText,
                          bool VerifyEachPass = true, bool DebugLogging = false);
 
+  /// {{@ Parse a textual pass pipeline description into a specific PassManager
+  ///
+  /// Automatic deduction of an appropriate pass manager stack is not supported.
+  /// For example, to insert a loop pass 'lpass' into a FunctinoPassManager,
+  /// this is the valid pipeline text:
+  ///
+  ///   function(lpass)
+  bool parsePassPipeline(CGSCCPassManager &CGPM, StringRef PipelineText,
+                         bool VerifyEachPass = true, bool DebugLogging = false);
+  bool parsePassPipeline(FunctionPassManager &FPM, StringRef PipelineText,
+                         bool VerifyEachPass = true, bool DebugLogging = false);
+  bool parsePassPipeline(LoopPassManager &LPM, StringRef PipelineText,
+                         bool VerifyEachPass = true, bool DebugLogging = false);
+  /// @}}
+
   /// Parse a textual alias analysis pipeline into the provided AA manager.
   ///
   /// The format of the textual AA pipeline is a comma separated list of AA
@@ -350,13 +389,139 @@ public:
   /// returns false.
   bool parseAAPipeline(AAManager &AA, StringRef PipelineText);
 
-private:
-  /// A struct to capture parsed pass pipeline names.
-  struct PipelineElement {
-    StringRef Name;
-    std::vector<PipelineElement> InnerPipeline;
-  };
+  /// \brief Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding passes that perform peephole
+  /// optimizations similar to the instruction combiner. These passes will be
+  /// inserted after each instance of the instruction combiner pass.
+  void registerPeepholeEPCallback(
+      const std::function<void(FunctionPassManager &, OptimizationLevel)> &C) {
+    PeepholeEPCallbacks.push_back(C);
+  }
+
+  /// \brief Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding late loop canonicalization and
+  /// simplification passes. This is the last point in the loop optimization
+  /// pipeline before loop deletion. Each pass added
+  /// here must be an instance of LoopPass.
+  /// This is the place to add passes that can remove loops, such as target-
+  /// specific loop idiom recognition.
+  void registerLateLoopOptimizationsEPCallback(
+      const std::function<void(LoopPassManager &, OptimizationLevel)> &C) {
+    LateLoopOptimizationsEPCallbacks.push_back(C);
+  }
+
+  /// \brief Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding loop passes to the end of the loop
+  /// optimizer.
+  void registerLoopOptimizerEndEPCallback(
+      const std::function<void(LoopPassManager &, OptimizationLevel)> &C) {
+    LoopOptimizerEndEPCallbacks.push_back(C);
+  }
+
+  /// \brief Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding optimization passes after most of the
+  /// main optimizations, but before the last cleanup-ish optimizations.
+  void registerScalarOptimizerLateEPCallback(
+      const std::function<void(FunctionPassManager &, OptimizationLevel)> &C) {
+    ScalarOptimizerLateEPCallbacks.push_back(C);
+  }
+
+  /// \brief Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding CallGraphSCC passes at the end of the
+  /// main CallGraphSCC passes and before any function simplification passes run
+  /// by CGPassManager.
+  void registerCGSCCOptimizerLateEPCallback(
+      const std::function<void(CGSCCPassManager &, OptimizationLevel)> &C) {
+    CGSCCOptimizerLateEPCallbacks.push_back(C);
+  }
+
+  /// \brief Register a callback for a default optimizer pipeline extension
+  /// point
+  ///
+  /// This extension point allows adding optimization passes before the
+  /// vectorizer and other highly target specific optimization passes are
+  /// executed.
+  void registerVectorizerStartEPCallback(
+      const std::function<void(FunctionPassManager &, OptimizationLevel)> &C) {
+    VectorizerStartEPCallbacks.push_back(C);
+  }
+
+  /// \brief Register a callback for parsing an AliasAnalysis Name to populate
+  /// the given AAManager \p AA
+  void registerParseAACallback(
+      const std::function<bool(StringRef Name, AAManager &AA)> &C) {
+    AAParsingCallbacks.push_back(C);
+  }
+
+  /// {{@ Register callbacks for analysis registration with this PassBuilder
+  /// instance.
+  /// Callees register their analyses with the given AnalysisManager objects.
+  void registerAnalysisRegistrationCallback(
+      const std::function<void(CGSCCAnalysisManager &)> &C) {
+    CGSCCAnalysisRegistrationCallbacks.push_back(C);
+  }
+  void registerAnalysisRegistrationCallback(
+      const std::function<void(FunctionAnalysisManager &)> &C) {
+    FunctionAnalysisRegistrationCallbacks.push_back(C);
+  }
+  void registerAnalysisRegistrationCallback(
+      const std::function<void(LoopAnalysisManager &)> &C) {
+    LoopAnalysisRegistrationCallbacks.push_back(C);
+  }
+  void registerAnalysisRegistrationCallback(
+      const std::function<void(ModuleAnalysisManager &)> &C) {
+    ModuleAnalysisRegistrationCallbacks.push_back(C);
+  }
+  /// @}}
+
+  /// {{@ Register pipeline parsing callbacks with this pass builder instance.
+  /// Using these callbacks, callers can parse both a single pass name, as well
+  /// as entire sub-pipelines, and populate the PassManager instance
+  /// accordingly.
+  void registerPipelineParsingCallback(
+      const std::function<bool(StringRef Name, CGSCCPassManager &,
+                               ArrayRef<PipelineElement>)> &C) {
+    CGSCCPipelineParsingCallbacks.push_back(C);
+  }
+  void registerPipelineParsingCallback(
+      const std::function<bool(StringRef Name, FunctionPassManager &,
+                               ArrayRef<PipelineElement>)> &C) {
+    FunctionPipelineParsingCallbacks.push_back(C);
+  }
+  void registerPipelineParsingCallback(
+      const std::function<bool(StringRef Name, LoopPassManager &,
+                               ArrayRef<PipelineElement>)> &C) {
+    LoopPipelineParsingCallbacks.push_back(C);
+  }
+  void registerPipelineParsingCallback(
+      const std::function<bool(StringRef Name, ModulePassManager &,
+                               ArrayRef<PipelineElement>)> &C) {
+    ModulePipelineParsingCallbacks.push_back(C);
+  }
+  /// @}}
+
+  /// \brief Register a callback for a top-level pipeline entry.
+  ///
+  /// If the PassManager type is not given at the top level of the pipeline
+  /// text, this Callback should be used to determine the appropriate stack of
+  /// PassManagers and populate the passed ModulePassManager.
+  void registerParseTopLevelPipelineCallback(
+      const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>,
+                               bool VerifyEachPass, bool DebugLogging)> &C) {
+    TopLevelPipelineParsingCallbacks.push_back(C);
+  }
 
+private:
   static Optional<std::vector<PipelineElement>>
   parsePipelineText(StringRef Text);
 
@@ -382,7 +547,106 @@ private:
   bool parseModulePassPipeline(ModulePassManager &MPM,
                                ArrayRef<PipelineElement> Pipeline,
                                bool VerifyEachPass, bool DebugLogging);
+
+  void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
+                         OptimizationLevel Level, bool RunProfileGen,
+                         std::string ProfileGenFile,
+                         std::string ProfileUseFile);
+
+  void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel);
+
+  // Extension Point callbacks
+  SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
+      PeepholeEPCallbacks;
+  SmallVector<std::function<void(LoopPassManager &, OptimizationLevel)>, 2>
+      LateLoopOptimizationsEPCallbacks;
+  SmallVector<std::function<void(LoopPassManager &, OptimizationLevel)>, 2>
+      LoopOptimizerEndEPCallbacks;
+  SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
+      ScalarOptimizerLateEPCallbacks;
+  SmallVector<std::function<void(CGSCCPassManager &, OptimizationLevel)>, 2>
+      CGSCCOptimizerLateEPCallbacks;
+  SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
+      VectorizerStartEPCallbacks;
+  // Module callbacks
+  SmallVector<std::function<void(ModuleAnalysisManager &)>, 2>
+      ModuleAnalysisRegistrationCallbacks;
+  SmallVector<std::function<bool(StringRef, ModulePassManager &,
+                                 ArrayRef<PipelineElement>)>,
+              2>
+      ModulePipelineParsingCallbacks;
+  SmallVector<std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>,
+                                 bool VerifyEachPass, bool DebugLogging)>,
+              2>
+      TopLevelPipelineParsingCallbacks;
+  // CGSCC callbacks
+  SmallVector<std::function<void(CGSCCAnalysisManager &)>, 2>
+      CGSCCAnalysisRegistrationCallbacks;
+  SmallVector<std::function<bool(StringRef, CGSCCPassManager &,
+                                 ArrayRef<PipelineElement>)>,
+              2>
+      CGSCCPipelineParsingCallbacks;
+  // Function callbacks
+  SmallVector<std::function<void(FunctionAnalysisManager &)>, 2>
+      FunctionAnalysisRegistrationCallbacks;
+  SmallVector<std::function<bool(StringRef, FunctionPassManager &,
+                                 ArrayRef<PipelineElement>)>,
+              2>
+      FunctionPipelineParsingCallbacks;
+  // Loop callbacks
+  SmallVector<std::function<void(LoopAnalysisManager &)>, 2>
+      LoopAnalysisRegistrationCallbacks;
+  SmallVector<std::function<bool(StringRef, LoopPassManager &,
+                                 ArrayRef<PipelineElement>)>,
+              2>
+      LoopPipelineParsingCallbacks;
+  // AA callbacks
+  SmallVector<std::function<bool(StringRef Name, AAManager &AA)>, 2>
+      AAParsingCallbacks;
 };
+
+/// This utility template takes care of adding require<> and invalidate<>
+/// passes for an analysis to a given \c PassManager. It is intended to be used
+/// during parsing of a pass pipeline when parsing a single PipelineName.
+/// When registering a new function analysis FancyAnalysis with the pass
+/// pipeline name "fancy-analysis", a matching ParsePipelineCallback could look
+/// like this:
+///
+/// static bool parseFunctionPipeline(StringRef Name, FunctionPassManager &FPM,
+///                                   ArrayRef<PipelineElement> P) {
+///   if (parseAnalysisUtilityPasses<FancyAnalysis>("fancy-analysis", Name,
+///                                                 FPM))
+///     return true;
+///   return false;
+/// }
+template <typename AnalysisT, typename IRUnitT, typename AnalysisManagerT,
+          typename... ExtraArgTs>
+bool parseAnalysisUtilityPasses(
+    StringRef AnalysisName, StringRef PipelineName,
+    PassManager<IRUnitT, AnalysisManagerT, ExtraArgTs...> &PM) {
+  if (!PipelineName.endswith(">"))
+    return false;
+  // See if this is an invalidate<> pass name
+  if (PipelineName.startswith("invalidate<")) {
+    PipelineName = PipelineName.substr(11, PipelineName.size() - 12);
+    if (PipelineName != AnalysisName)
+      return false;
+    PM.addPass(InvalidateAnalysisPass<AnalysisT>());
+    return true;
+  }
+
+  // See if this is a require<> pass name
+  if (PipelineName.startswith("require<")) {
+    PipelineName = PipelineName.substr(8, PipelineName.size() - 9);
+    if (PipelineName != AnalysisName)
+      return false;
+    PM.addPass(RequireAnalysisPass<AnalysisT, IRUnitT, AnalysisManagerT,
+                                   ExtraArgTs...>());
+    return true;
+  }
+
+  return false;
+}
 }
 
 #endif
diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h
index a6b2850ccd228..772187f70153c 100644
--- a/include/llvm/ProfileData/InstrProf.h
+++ b/include/llvm/ProfileData/InstrProf.h
@@ -249,9 +249,8 @@ void annotateValueSite(Module &M, Instruction &Inst,
 
 /// Same as the above interface but using an ArrayRef, as well as \p Sum.
 void annotateValueSite(Module &M, Instruction &Inst,
-                       ArrayRef<InstrProfValueData> VDs,
-                       uint64_t Sum, InstrProfValueKind ValueKind,
-                       uint32_t MaxMDCount);
+                       ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
+                       InstrProfValueKind ValueKind, uint32_t MaxMDCount);
 
 /// Extract the value profile data from \p Inst which is annotated with
 /// value profile meta data. Return false if there is no value data annotated,
@@ -582,34 +581,27 @@ struct InstrProfValueSiteRecord {
 
   /// Merge data from another InstrProfValueSiteRecord
   /// Optionally scale merged counts by \p Weight.
-  void merge(SoftInstrProfErrors &SIPE, InstrProfValueSiteRecord &Input,
-             uint64_t Weight = 1);
+  void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
+             function_ref<void(instrprof_error)> Warn);
   /// Scale up value profile data counts.
-  void scale(SoftInstrProfErrors &SIPE, uint64_t Weight);
+  void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
 };
 
 /// Profiling information for a single function.
 struct InstrProfRecord {
-  StringRef Name;
-  uint64_t Hash;
   std::vector<uint64_t> Counts;
-  SoftInstrProfErrors SIPE;
 
   InstrProfRecord() = default;
-  InstrProfRecord(StringRef Name, uint64_t Hash, std::vector<uint64_t> Counts)
-      : Name(Name), Hash(Hash), Counts(std::move(Counts)) {}
+  InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
   InstrProfRecord(InstrProfRecord &&) = default;
   InstrProfRecord(const InstrProfRecord &RHS)
-      : Name(RHS.Name), Hash(RHS.Hash), Counts(RHS.Counts), SIPE(RHS.SIPE),
+      : Counts(RHS.Counts),
         ValueData(RHS.ValueData
                       ? llvm::make_unique<ValueProfData>(*RHS.ValueData)
                       : nullptr) {}
   InstrProfRecord &operator=(InstrProfRecord &&) = default;
   InstrProfRecord &operator=(const InstrProfRecord &RHS) {
-    Name = RHS.Name;
-    Hash = RHS.Hash;
     Counts = RHS.Counts;
-    SIPE = RHS.SIPE;
     if (!RHS.ValueData) {
       ValueData = nullptr;
       return *this;
@@ -626,7 +618,6 @@ struct InstrProfRecord {
   /// Return the number of value profile kinds with non-zero number
   /// of profile sites.
   inline uint32_t getNumValueKinds() const;
-
   /// Return the number of instrumented sites for ValueKind.
   inline uint32_t getNumValueSites(uint32_t ValueKind) const;
 
@@ -661,11 +652,12 @@ struct InstrProfRecord {
 
   /// Merge the counts in \p Other into this one.
   /// Optionally scale merged counts by \p Weight.
-  void merge(InstrProfRecord &Other, uint64_t Weight = 1);
+  void merge(InstrProfRecord &Other, uint64_t Weight,
+             function_ref<void(instrprof_error)> Warn);
 
   /// Scale up profile counts (including value profile data) by
   /// \p Weight.
-  void scale(uint64_t Weight);
+  void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
 
   /// Sort value profile data (per site) by count.
   void sortValueData() {
@@ -683,9 +675,6 @@ struct InstrProfRecord {
   /// Clear value data entries
   void clearValueData() { ValueData = nullptr; }
 
-  /// Get the error contained within the record's soft error counter.
-  Error takeError() { return SIPE.takeError(); }
-
 private:
   struct ValueProfData {
     std::vector<InstrProfValueSiteRecord> IndirectCallSites;
@@ -737,11 +726,23 @@ private:
 
   // Merge Value Profile data from Src record to this record for ValueKind.
   // Scale merged value counts by \p Weight.
-  void mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
-                          uint64_t Weight);
+  void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
+                          uint64_t Weight,
+                          function_ref<void(instrprof_error)> Warn);
 
   // Scale up value profile data count.
-  void scaleValueProfData(uint32_t ValueKind, uint64_t Weight);
+  void scaleValueProfData(uint32_t ValueKind, uint64_t Weight,
+                          function_ref<void(instrprof_error)> Warn);
+};
+
+struct NamedInstrProfRecord : InstrProfRecord {
+  StringRef Name;
+  uint64_t Hash;
+
+  NamedInstrProfRecord() = default;
+  NamedInstrProfRecord(StringRef Name, uint64_t Hash,
+                       std::vector<uint64_t> Counts)
+      : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
 };
 
 uint32_t InstrProfRecord::getNumValueKinds() const {
@@ -753,11 +754,8 @@ uint32_t InstrProfRecord::getNumValueKinds() const {
 
 uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
   uint32_t N = 0;
-  const std::vector<InstrProfValueSiteRecord> &SiteRecords =
-      getValueSitesForKind(ValueKind);
-  for (auto &SR : SiteRecords) {
+  for (auto &SR : getValueSitesForKind(ValueKind))
     N += SR.ValueData.size();
-  }
   return N;
 }
 
diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h
index 8163ca1592098..424360e0f7655 100644
--- a/include/llvm/ProfileData/InstrProfReader.h
+++ b/include/llvm/ProfileData/InstrProfReader.h
@@ -40,9 +40,9 @@ class InstrProfReader;
 
 /// A file format agnostic iterator over profiling data.
 class InstrProfIterator : public std::iterator<std::input_iterator_tag,
-                                               InstrProfRecord> {
+                                               NamedInstrProfRecord> {
   InstrProfReader *Reader = nullptr;
-  InstrProfRecord Record;
+  value_type Record;
 
   void Increment();
 
@@ -53,12 +53,12 @@ public:
   InstrProfIterator &operator++() { Increment(); return *this; }
   bool operator==(const InstrProfIterator &RHS) { return Reader == RHS.Reader; }
   bool operator!=(const InstrProfIterator &RHS) { return Reader != RHS.Reader; }
-  InstrProfRecord &operator*() { return Record; }
-  InstrProfRecord *operator->() { return &Record; }
+  value_type &operator*() { return Record; }
+  value_type *operator->() { return &Record; }
 };
 
 /// Base class and interface for reading profiling data of any known instrprof
-/// format. Provides an iterator over InstrProfRecords.
+/// format. Provides an iterator over NamedInstrProfRecords.
 class InstrProfReader {
   instrprof_error LastError = instrprof_error::success;
 
@@ -70,7 +70,7 @@ public:
   virtual Error readHeader() = 0;
 
   /// Read a single record.
-  virtual Error readNextRecord(InstrProfRecord &Record) = 0;
+  virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0;
 
   /// Iterator over profile data.
   InstrProfIterator begin() { return InstrProfIterator(this); }
@@ -161,7 +161,7 @@ public:
   Error readHeader() override;
 
   /// Read a single record.
-  Error readNextRecord(InstrProfRecord &Record) override;
+  Error readNextRecord(NamedInstrProfRecord &Record) override;
 
   InstrProfSymtab &getSymtab() override {
     assert(Symtab.get());
@@ -209,7 +209,7 @@ public:
 
   static bool hasFormat(const MemoryBuffer &DataBuffer);
   Error readHeader() override;
-  Error readNextRecord(InstrProfRecord &Record) override;
+  Error readNextRecord(NamedInstrProfRecord &Record) override;
 
   bool isIRLevelProfile() const override {
     return (Version & VARIANT_MASK_IR_PROF) != 0;
@@ -243,8 +243,8 @@ private:
     return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
   }
 
-  Error readName(InstrProfRecord &Record);
-  Error readFuncHash(InstrProfRecord &Record);
+  Error readName(NamedInstrProfRecord &Record);
+  Error readFuncHash(NamedInstrProfRecord &Record);
   Error readRawCounts(InstrProfRecord &Record);
   Error readValueProfilingData(InstrProfRecord &Record);
   bool atEnd() const { return Data == DataEnd; }
@@ -281,7 +281,7 @@ enum class HashT : uint32_t;
 /// Trait for lookups into the on-disk hash table for the binary instrprof
 /// format.
 class InstrProfLookupTrait {
-  std::vector<InstrProfRecord> DataBuffer;
+  std::vector<NamedInstrProfRecord> DataBuffer;
   IndexedInstrProf::HashT HashType;
   unsigned FormatVersion;
   // Endianness of the input value profile data.
@@ -293,7 +293,7 @@ public:
   InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
       : HashType(HashType), FormatVersion(FormatVersion) {}
 
-  using data_type = ArrayRef<InstrProfRecord>;
+  using data_type = ArrayRef<NamedInstrProfRecord>;
 
   using internal_key_type = StringRef;
   using external_key_type = StringRef;
@@ -334,11 +334,11 @@ struct InstrProfReaderIndexBase {
 
   // Read all the profile records with the same key pointed to the current
   // iterator.
-  virtual Error getRecords(ArrayRef<InstrProfRecord> &Data) = 0;
+  virtual Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) = 0;
 
   // Read all the profile records with the key equal to FuncName
   virtual Error getRecords(StringRef FuncName,
-                                     ArrayRef<InstrProfRecord> &Data) = 0;
+                                     ArrayRef<NamedInstrProfRecord> &Data) = 0;
   virtual void advanceToNextKey() = 0;
   virtual bool atEnd() const = 0;
   virtual void setValueProfDataEndianness(support::endianness Endianness) = 0;
@@ -364,9 +364,9 @@ public:
                        IndexedInstrProf::HashT HashType, uint64_t Version);
   ~InstrProfReaderIndex() override = default;
 
-  Error getRecords(ArrayRef<InstrProfRecord> &Data) override;
+  Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) override;
   Error getRecords(StringRef FuncName,
-                   ArrayRef<InstrProfRecord> &Data) override;
+                   ArrayRef<NamedInstrProfRecord> &Data) override;
   void advanceToNextKey() override { RecordIterator++; }
 
   bool atEnd() const override {
@@ -419,10 +419,9 @@ public:
   /// Read the file header.
   Error readHeader() override;
   /// Read a single record.
-  Error readNextRecord(InstrProfRecord &Record) override;
+  Error readNextRecord(NamedInstrProfRecord &Record) override;
 
-  /// Return the pointer to InstrProfRecord associated with FuncName
-  /// and FuncHash
+  /// Return the NamedInstrProfRecord associated with FuncName and FuncHash
   Expected<InstrProfRecord> getInstrProfRecord(StringRef FuncName,
                                                uint64_t FuncHash);
 
diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h
index fff10af30295a..8107ab386fe23 100644
--- a/include/llvm/ProfileData/InstrProfWriter.h
+++ b/include/llvm/ProfileData/InstrProfWriter.h
@@ -33,7 +33,7 @@ class raw_fd_ostream;
 
 class InstrProfWriter {
 public:
-  using ProfilingData = SmallDenseMap<uint64_t, InstrProfRecord, 1>;
+  using ProfilingData = SmallDenseMap<uint64_t, InstrProfRecord>;
   enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel };
 
 private:
@@ -50,10 +50,15 @@ public:
   /// Add function counts for the given function. If there are already counts
   /// for this function and the hash and number of counts match, each counter is
   /// summed. Optionally scale counts by \p Weight.
-  Error addRecord(InstrProfRecord &&I, uint64_t Weight = 1);
+  void addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
+                 function_ref<void(Error)> Warn);
+  void addRecord(NamedInstrProfRecord &&I, function_ref<void(Error)> Warn) {
+    addRecord(std::move(I), 1, Warn);
+  }
 
   /// Merge existing function counts from the given writer.
-  Error mergeRecordsFromWriter(InstrProfWriter &&IPW);
+  void mergeRecordsFromWriter(InstrProfWriter &&IPW,
+                              function_ref<void(Error)> Warn);
 
   /// Write the profile to \c OS
   void write(raw_fd_ostream &OS);
@@ -62,7 +67,8 @@ public:
   Error writeText(raw_fd_ostream &OS);
 
   /// Write \c Record in text format to \c OS
-  static void writeRecordInText(const InstrProfRecord &Record,
+  static void writeRecordInText(StringRef Name, uint64_t Hash,
+                                const InstrProfRecord &Counters,
                                 InstrProfSymtab &Symtab, raw_fd_ostream &OS);
 
   /// Write the profile, returning the raw data. For testing.
@@ -85,6 +91,8 @@ public:
   void setOutputSparse(bool Sparse);
 
 private:
+  void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I,
+                 uint64_t Weight, function_ref<void(Error)> Warn);
   bool shouldEncodeData(const ProfilingData &PD);
   void writeImpl(ProfOStream &OS);
 };
diff --git a/include/llvm/ProfileData/ProfileCommon.h b/include/llvm/ProfileData/ProfileCommon.h
index 987e3160ccae2..51b065bcdb700 100644
--- a/include/llvm/ProfileData/ProfileCommon.h
+++ b/include/llvm/ProfileData/ProfileCommon.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/IR/ProfileSummary.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cstdint>
@@ -27,8 +28,6 @@
 
 namespace llvm {
 
-struct InstrProfRecord;
-
 namespace sampleprof {
 
 class FunctionSamples;
diff --git a/include/llvm/Support/BlockFrequency.h b/include/llvm/Support/BlockFrequency.h
index 1b45cc52973f6..2e75cbdd29c16 100644
--- a/include/llvm/Support/BlockFrequency.h
+++ b/include/llvm/Support/BlockFrequency.h
@@ -71,6 +71,10 @@ public:
   bool operator>=(BlockFrequency RHS) const {
     return Frequency >= RHS.Frequency;
   }
+
+  bool operator==(BlockFrequency RHS) const {
+    return Frequency == RHS.Frequency;
+  }
 };
 
 }
diff --git a/include/llvm/Support/Compiler.h b/include/llvm/Support/Compiler.h
index be9e465400165..b19e37235df57 100644
--- a/include/llvm/Support/Compiler.h
+++ b/include/llvm/Support/Compiler.h
@@ -493,4 +493,14 @@ void AnnotateIgnoreWritesEnd(const char *file, int line);
 #define LLVM_THREAD_LOCAL
 #endif
 
+/// \macro LLVM_ENABLE_EXCEPTIONS
+/// \brief Whether LLVM is built with exception support.
+#if __has_feature(cxx_exceptions)
+#define LLVM_ENABLE_EXCEPTIONS 1
+#elif defined(__GNUC__) && defined(__EXCEPTIONS)
+#define LLVM_ENABLE_EXCEPTIONS 1
+#elif defined(_MSC_VER) && defined(_CPPUNWIND)
+#define LLVM_ENABLE_EXCEPTIONS 1
+#endif
+
 #endif
diff --git a/include/llvm/Support/DynamicLibrary.h b/include/llvm/Support/DynamicLibrary.h
index a8874a10d461a..469d5dfad0627 100644
--- a/include/llvm/Support/DynamicLibrary.h
+++ b/include/llvm/Support/DynamicLibrary.h
@@ -88,6 +88,22 @@ namespace sys {
       return !getPermanentLibrary(Filename, ErrMsg).isValid();
     }
 
+    enum SearchOrdering {
+      /// SO_Linker - Search as a call to dlsym(dlopen(NULL)) would when
+      /// DynamicLibrary::getPermanentLibrary(NULL) has been called or
+      /// search the list of explcitly loaded symbols if not.
+      SO_Linker,
+      /// SO_LoadedFirst - Search all loaded libraries, then as SO_Linker would.
+      SO_LoadedFirst,
+      /// SO_LoadedLast - Search as SO_Linker would, then loaded libraries.
+      /// Only useful to search if libraries with RTLD_LOCAL have been added.
+      SO_LoadedLast,
+      /// SO_LoadOrder - Or this in to search libraries in the ordered loaded.
+      /// The default bahaviour is to search loaded libraries in reverse.
+      SO_LoadOrder = 4
+    };
+    static SearchOrdering SearchOrder; // = SO_Linker
+
     /// This function will search through all previously loaded dynamic
     /// libraries for the symbol \p symbolName. If it is found, the address of
     /// that symbol is returned. If not, null is returned. Note that this will
diff --git a/include/llvm/Support/ErrorHandling.h b/include/llvm/Support/ErrorHandling.h
index 7c1edd8015712..b45f6348390e2 100644
--- a/include/llvm/Support/ErrorHandling.h
+++ b/include/llvm/Support/ErrorHandling.h
@@ -78,12 +78,48 @@ LLVM_ATTRIBUTE_NORETURN void report_fatal_error(StringRef reason,
 LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const Twine &reason,
                                                 bool gen_crash_diag = true);
 
-  /// This function calls abort(), and prints the optional message to stderr.
-  /// Use the llvm_unreachable macro (that adds location info), instead of
-  /// calling this function directly.
-  LLVM_ATTRIBUTE_NORETURN void
-  llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr,
-                            unsigned line=0);
+/// Installs a new bad alloc error handler that should be used whenever a
+/// bad alloc error, e.g. failing malloc/calloc, is encountered by LLVM.
+///
+/// The user can install a bad alloc handler, in order to define the behavior
+/// in case of failing allocations, e.g. throwing an exception. Note that this
+/// handler must not trigger any additional allocations itself.
+///
+/// If no error handler is installed the default is to print the error message
+/// to stderr, and call exit(1).  If an error handler is installed then it is
+/// the handler's responsibility to log the message, it will no longer be
+/// printed to stderr.  If the error handler returns, then exit(1) will be
+/// called.
+///
+///
+/// \param user_data - An argument which will be passed to the installed error
+/// handler.
+void install_bad_alloc_error_handler(fatal_error_handler_t handler,
+                                     void *user_data = nullptr);
+
+/// Restores default bad alloc error handling behavior.
+void remove_bad_alloc_error_handler();
+
+/// Reports a bad alloc error, calling any user defined bad alloc
+/// error handler. In contrast to the generic 'report_fatal_error'
+/// functions, this function is expected to return, e.g. the user
+/// defined error handler throws an exception.
+///
+/// Note: When throwing an exception in the bad alloc handler, make sure that
+/// the following unwind succeeds, e.g. do not trigger additional allocations
+/// in the unwind chain.
+///
+/// If no error handler is installed (default), then a bad_alloc exception
+/// is thrown if LLVM is compiled with exception support, otherwise an assertion
+/// is called.
+void report_bad_alloc_error(const char *Reason, bool GenCrashDiag = true);
+
+/// This function calls abort(), and prints the optional message to stderr.
+/// Use the llvm_unreachable macro (that adds location info), instead of
+/// calling this function directly.
+LLVM_ATTRIBUTE_NORETURN void
+llvm_unreachable_internal(const char *msg = nullptr, const char *file = nullptr,
+                          unsigned line = 0);
 }
 
 /// Marks that the current location is not supposed to be reachable.
diff --git a/include/llvm/Support/GenericDomTreeConstruction.h b/include/llvm/Support/GenericDomTreeConstruction.h
index 9edf03aa36210..a0fec668e05ca 100644
--- a/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/include/llvm/Support/GenericDomTreeConstruction.h
@@ -32,6 +32,20 @@
 namespace llvm {
 namespace DomTreeBuilder {
 
+template <typename NodePtr, bool Inverse>
+struct ChildrenGetter {
+  static auto Get(NodePtr N) -> decltype(reverse(children<NodePtr>(N))) {
+    return reverse(children<NodePtr>(N));
+  }
+};
+
+template <typename NodePtr>
+struct ChildrenGetter<NodePtr, true> {
+  static auto Get(NodePtr N) -> decltype(inverse_children<NodePtr>(N)) {
+    return inverse_children<NodePtr>(N);
+  }
+};
+
 // Information record used by Semi-NCA during tree construction.
 template <typename NodeT>
 struct SemiNCAInfo {
@@ -45,6 +59,7 @@ struct SemiNCAInfo {
     unsigned Semi = 0;
     NodePtr Label = nullptr;
     NodePtr IDom = nullptr;
+    SmallVector<NodePtr, 2> ReverseChildren;
   };
 
   std::vector<NodePtr> NumToNode;
@@ -79,66 +94,49 @@ struct SemiNCAInfo {
         .get();
   }
 
-  // External storage for depth first iterator that reuses the info lookup map
-  // SemiNCAInfo already has. We don't have a set, but a map instead, so we are
-  // converting the one argument insert calls.
-  struct df_iterator_dom_storage {
-   public:
-    using BaseSet = decltype(NodeToInfo);
-    df_iterator_dom_storage(BaseSet &Storage) : Storage(Storage) {}
-
-    using iterator = typename BaseSet::iterator;
-    std::pair<iterator, bool> insert(NodePtr N) {
-      return Storage.insert({N, InfoRec()});
-    }
-    void completed(NodePtr) {}
-
-   private:
-    BaseSet &Storage;
-  };
-
-  df_iterator_dom_storage getStorage() { return {NodeToInfo}; }
+  static bool AlwaysDescend(NodePtr, NodePtr) { return true; }
 
-  unsigned runReverseDFS(NodePtr V, unsigned N) {
-    auto DFStorage = getStorage();
+  // Custom DFS implementation which can skip nodes based on a provided
+  // predicate. It also collects ReverseChildren so that we don't have to spend
+  // time getting predecessors in SemiNCA.
+  template <bool Inverse, typename DescendCondition>
+  unsigned runDFS(NodePtr V, unsigned LastNum, DescendCondition Condition,
+                  unsigned AttachToNum) {
+    assert(V);
+    SmallVector<NodePtr, 64> WorkList = {V};
+    if (NodeToInfo.count(V) != 0) NodeToInfo[V].Parent = AttachToNum;
 
-    bool IsChildOfArtificialExit = (N != 0);
-    for (auto I = idf_ext_begin(V, DFStorage), E = idf_ext_end(V, DFStorage);
-         I != E; ++I) {
-      NodePtr BB = *I;
+    while (!WorkList.empty()) {
+      const NodePtr BB = WorkList.pop_back_val();
       auto &BBInfo = NodeToInfo[BB];
-      BBInfo.DFSNum = BBInfo.Semi = ++N;
+
+      // Visited nodes always have positive DFS numbers.
+      if (BBInfo.DFSNum != 0) continue;
+      BBInfo.DFSNum = BBInfo.Semi = ++LastNum;
       BBInfo.Label = BB;
-      // Set the parent to the top of the visited stack.  The stack includes us,
-      // and is 1 based, so we subtract to account for both of these.
-      if (I.getPathLength() > 1)
-        BBInfo.Parent = NodeToInfo[I.getPath(I.getPathLength() - 2)].DFSNum;
-      NumToNode.push_back(BB);  // NumToNode[n] = V;
+      NumToNode.push_back(BB);
+
+      for (const NodePtr Succ : ChildrenGetter<NodePtr, Inverse>::Get(BB)) {
+        const auto SIT = NodeToInfo.find(Succ);
+        // Don't visit nodes more than once but remember to collect
+        // RerverseChildren.
+        if (SIT != NodeToInfo.end() && SIT->second.DFSNum != 0) {
+          if (Succ != BB) SIT->second.ReverseChildren.push_back(BB);
+          continue;
+        }
 
-      if (IsChildOfArtificialExit)
-        BBInfo.Parent = 1;
+        if (!Condition(BB, Succ)) continue;
 
-      IsChildOfArtificialExit = false;
+        // It's fine to add Succ to the map, because we know that it will be
+        // visited later.
+        auto &SuccInfo = NodeToInfo[Succ];
+        WorkList.push_back(Succ);
+        SuccInfo.Parent = LastNum;
+        SuccInfo.ReverseChildren.push_back(BB);
+      }
     }
-    return N;
-  }
-
-  unsigned runForwardDFS(NodePtr V, unsigned N) {
-    auto DFStorage = getStorage();
 
-    for (auto I = df_ext_begin(V, DFStorage), E = df_ext_end(V, DFStorage);
-         I != E; ++I) {
-      NodePtr BB = *I;
-      auto &BBInfo = NodeToInfo[BB];
-      BBInfo.DFSNum = BBInfo.Semi = ++N;
-      BBInfo.Label = BB;
-      // Set the parent to the top of the visited stack.  The stack includes us,
-      // and is 1 based, so we subtract to account for both of these.
-      if (I.getPathLength() > 1)
-        BBInfo.Parent = NodeToInfo[I.getPath(I.getPathLength() - 2)].DFSNum;
-      NumToNode.push_back(BB);  // NumToNode[n] = V;
-    }
-    return N;
+    return LastNum;
   }
 
   NodePtr eval(NodePtr VIn, unsigned LastLinked) {
@@ -181,31 +179,14 @@ struct SemiNCAInfo {
 
   template <typename NodeType>
   void runSemiNCA(DomTreeT &DT, unsigned NumBlocks) {
-    unsigned N = 0;
-    NumToNode.push_back(nullptr);
-
-    bool MultipleRoots = (DT.Roots.size() > 1);
-    if (MultipleRoots) {
-      auto &BBInfo = NodeToInfo[nullptr];
-      BBInfo.DFSNum = BBInfo.Semi = ++N;
-      BBInfo.Label = nullptr;
-
-      NumToNode.push_back(nullptr); // NumToNode[n] = V;
-    }
-
     // Step #1: Number blocks in depth-first order and initialize variables used
     // in later stages of the algorithm.
-    if (DT.isPostDominator()){
-      for (unsigned i = 0, e = static_cast<unsigned>(DT.Roots.size());
-           i != e; ++i)
-        N = runReverseDFS(DT.Roots[i], N);
-    } else {
-      N = runForwardDFS(DT.Roots[0], N);
-    }
+    const unsigned N = doFullDFSWalk(DT, AlwaysDescend);
 
     // It might be that some blocks did not get a DFS number (e.g., blocks of
     // infinite loops). In these cases an artificial exit node is required.
-    MultipleRoots |= (DT.isPostDominator() && N != NumBlocks);
+    const bool MultipleRoots =
+        DT.Roots.size() > 1 || (DT.isPostDominator() && N != NumBlocks);
 
     // Initialize IDoms to spanning tree parents.
     for (unsigned i = 1; i <= N; ++i) {
@@ -221,7 +202,7 @@ struct SemiNCAInfo {
 
       // Initialize the semi dominator to point to the parent node.
       WInfo.Semi = WInfo.Parent;
-      for (const auto &N : inverse_children<NodeType>(W))
+      for (const auto &N : WInfo.ReverseChildren)
         if (NodeToInfo.count(N)) {  // Only if this predecessor is reachable!
           unsigned SemiU = NodeToInfo[eval(N, i + 1)].Semi;
           if (SemiU < WInfo.Semi)
@@ -279,14 +260,27 @@ struct SemiNCAInfo {
     }
   }
 
-  void doFullDFSWalk(const DomTreeT &DT) {
-    NumToNode.push_back(nullptr);
+  template <typename DescendCondition>
+  unsigned doFullDFSWalk(const DomTreeT &DT, DescendCondition DC) {
     unsigned Num = 0;
-    for (auto *Root : DT.Roots)
-      if (!DT.isPostDominator())
-        Num = runForwardDFS(Root, Num);
-      else
-        Num = runReverseDFS(Root, Num);
+    NumToNode.push_back(nullptr);
+
+    if (DT.Roots.size() > 1) {
+      auto &BBInfo = NodeToInfo[nullptr];
+      BBInfo.DFSNum = BBInfo.Semi = ++Num;
+      BBInfo.Label = nullptr;
+
+      NumToNode.push_back(nullptr);  // NumToNode[n] = V;
+    }
+
+    if (DT.isPostDominator()) {
+      for (auto *Root : DT.Roots) Num = runDFS<true>(Root, Num, DC, 1);
+    } else {
+      assert(DT.Roots.size() == 1);
+      Num = runDFS<false>(DT.Roots[0], Num, DC, Num);
+    }
+
+    return Num;
   }
 
   static void PrintBlockOrNullptr(raw_ostream &O, NodePtr Obj) {
@@ -299,7 +293,7 @@ struct SemiNCAInfo {
   // Checks if the tree contains all reachable nodes in the input graph.
   bool verifyReachability(const DomTreeT &DT) {
     clear();
-    doFullDFSWalk(DT);
+    doFullDFSWalk(DT, AlwaysDescend);
 
     for (auto &NodeToTN : DT.DomTreeNodes) {
       const TreeNodePtr TN = NodeToTN.second.get();
@@ -356,7 +350,7 @@ struct SemiNCAInfo {
   //     NCD(From, To) == IDom(To) or To.
   bool verifyNCD(const DomTreeT &DT) {
     clear();
-    doFullDFSWalk(DT);
+    doFullDFSWalk(DT, AlwaysDescend);
 
     for (auto &BlockToInfo : NodeToInfo) {
       auto &Info = BlockToInfo.second;
@@ -440,8 +434,9 @@ struct SemiNCAInfo {
       if (!BB || TN->getChildren().empty()) continue;
 
       clear();
-      NodeToInfo.insert({BB, {}});
-      doFullDFSWalk(DT);
+      doFullDFSWalk(DT, [BB](NodePtr From, NodePtr To) {
+        return From != BB && To != BB;
+      });
 
       for (TreeNodePtr Child : TN->getChildren())
         if (NodeToInfo.count(Child->getBlock()) != 0) {
@@ -473,8 +468,10 @@ struct SemiNCAInfo {
       const auto &Siblings = TN->getChildren();
       for (const TreeNodePtr N : Siblings) {
         clear();
-        NodeToInfo.insert({N->getBlock(), {}});
-        doFullDFSWalk(DT);
+        NodePtr BBN = N->getBlock();
+        doFullDFSWalk(DT, [BBN](NodePtr From, NodePtr To) {
+          return From != BBN && To != BBN;
+        });
 
         for (const TreeNodePtr S : Siblings) {
           if (S == N) continue;
diff --git a/include/llvm/Support/ReverseIteration.h b/include/llvm/Support/ReverseIteration.h
new file mode 100644
index 0000000000000..cb97b60f06dd9
--- /dev/null
+++ b/include/llvm/Support/ReverseIteration.h
@@ -0,0 +1,17 @@
+#ifndef LLVM_SUPPORT_REVERSEITERATION_H
+#define LLVM_SUPPORT_REVERSEITERATION_H
+
+#include "llvm/Config/abi-breaking.h"
+
+namespace llvm {
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+template <class T = void> struct ReverseIterate { static bool value; };
+#if LLVM_ENABLE_REVERSE_ITERATION
+template <class T> bool ReverseIterate<T>::value = true;
+#else
+template <class T> bool ReverseIterate<T>::value = false;
+#endif
+#endif
+}
+
+#endif
diff --git a/include/llvm/Support/UnicodeCharRanges.h b/include/llvm/Support/UnicodeCharRanges.h
index d4d4d8eb84a4b..4c655833b3967 100644
--- a/include/llvm/Support/UnicodeCharRanges.h
+++ b/include/llvm/Support/UnicodeCharRanges.h
@@ -18,11 +18,11 @@
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
+#define DEBUG_TYPE "unicode"
+
 namespace llvm {
 namespace sys {
 
-#define DEBUG_TYPE "unicode"
-
 /// \brief Represents a closed range of Unicode code points [Lower, Upper].
 struct UnicodeCharRange {
   uint32_t Lower;
@@ -99,10 +99,9 @@ private:
   const CharRanges Ranges;
 };
 
-#undef DEBUG_TYPE // "unicode"
-
 } // namespace sys
 } // namespace llvm
 
+#undef DEBUG_TYPE // "unicode"
 
 #endif // LLVM_SUPPORT_UNICODECHARRANGES_H
diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 3a3118139bcbd..178b08d7b8b7d 100644
--- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -64,6 +64,7 @@ def : GINodeEquiv<G_FREM, frem>;
 def : GINodeEquiv<G_FPOW, fpow>;
 def : GINodeEquiv<G_FEXP2, fexp2>;
 def : GINodeEquiv<G_FLOG2, flog2>;
+def : GINodeEquiv<G_INTRINSIC, intrinsic_wo_chain>;
 def : GINodeEquiv<G_BR, br>;
 
 // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern.
diff --git a/include/llvm/Target/TargetInstrInfo.h b/include/llvm/Target/TargetInstrInfo.h
index 2fc3ec996e7f6..1843a2eed9bff 100644
--- a/include/llvm/Target/TargetInstrInfo.h
+++ b/include/llvm/Target/TargetInstrInfo.h
@@ -1545,6 +1545,16 @@ public:
     return None;
   }
 
+  /// Return an array that contains the MMO target flag values and their
+  /// names.
+  ///
+  /// MIR Serialization is able to serialize only the MMO target flags that are
+  /// defined by this method.
+  virtual ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
+  getSerializableMachineMemOperandTargetFlags() const {
+    return None;
+  }
+
   /// Determines whether \p Inst is a tail call instruction. Override this
   /// method on targets that do not properly set MCID::Return and MCID::Call on
   /// tail call instructions."
diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h
index 964d6314b1270..60a03bdc182d5 100644
--- a/include/llvm/Target/TargetLowering.h
+++ b/include/llvm/Target/TargetLowering.h
@@ -415,7 +415,8 @@ public:
   virtual bool mergeStoresAfterLegalization() const { return false; }
 
   /// Returns if it's reasonable to merge stores to MemVT size.
-  virtual bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT) const {
+  virtual bool canMergeStoresTo(unsigned AS, EVT MemVT,
+                                const SelectionDAG &DAG) const {
     return true;
   }
 
@@ -2726,6 +2727,18 @@ public:
     return true;
   }
 
+  // Return true if it is profitable to combine a BUILD_VECTOR to a TRUNCATE.
+  // Example of such a combine:
+  // v4i32 build_vector((extract_elt V, 0),
+  //                    (extract_elt V, 2),
+  //                    (extract_elt V, 4),
+  //                    (extract_elt V, 6))
+  //  -->
+  // v4i32 truncate (bitcast V to v4i64)
+  virtual bool isDesirableToCombineBuildVectorToTruncate() const {
+    return false;
+  }
+
   /// Return true if the target has native support for the specified value type
   /// and it is 'desirable' to use the type for the given node type. e.g. On x86
   /// i16 is legal, but undesirable since i16 instruction encodings are longer
@@ -2815,6 +2828,9 @@ public:
     // TargetLowering::LowerCall that perform tail call conversions.
     bool IsTailCall = false;
 
+    // Is Call lowering done post SelectionDAG type legalization.
+    bool IsPostTypeLegalization = false;
+
     unsigned NumFixedArgs = -1;
     CallingConv::ID CallConv = CallingConv::C;
     SDValue Callee;
@@ -2937,6 +2953,11 @@ public:
       return *this;
     }
 
+    CallLoweringInfo &setIsPostTypeLegalization(bool Value=true) {
+      IsPostTypeLegalization = Value;
+      return *this;
+    }
+
     ArgListTy &getArgs() {
       return Args;
     }
@@ -3055,6 +3076,13 @@ public:
     return Chain;
   }
 
+  /// This callback is used to inspect load/store instructions and add
+  /// target-specific MachineMemOperand flags to them.  The default
+  /// implementation does nothing.
+  virtual MachineMemOperand::Flags getMMOFlags(const Instruction &I) const {
+    return MachineMemOperand::MONone;
+  }
+
   /// This callback is invoked by the type legalizer to legalize nodes with an
   /// illegal operand type but legal result types.  It replaces the
   /// LowerOperation callback in the type Legalizer.  The reason we can not do
diff --git a/include/llvm/Transforms/Scalar/GVN.h b/include/llvm/Transforms/Scalar/GVN.h
index 4c585a20021cd..f25ab40640dfc 100644
--- a/include/llvm/Transforms/Scalar/GVN.h
+++ b/include/llvm/Transforms/Scalar/GVN.h
@@ -68,21 +68,6 @@ public:
   class ValueTable {
     DenseMap<Value *, uint32_t> valueNumbering;
     DenseMap<Expression, uint32_t> expressionNumbering;
-
-    // Expressions is the vector of Expression. ExprIdx is the mapping from
-    // value number to the index of Expression in Expressions. We use it
-    // instead of a DenseMap because filling such mapping is faster than
-    // filling a DenseMap and the compile time is a little better.
-    uint32_t nextExprNumber;
-    std::vector<Expression> Expressions;
-    std::vector<uint32_t> ExprIdx;
-    // Value number to PHINode mapping. Used for phi-translate in scalarpre.
-    DenseMap<uint32_t, PHINode *> NumberingPhi;
-    // Cache for phi-translate in scalarpre.
-    typedef DenseMap<std::pair<uint32_t, const BasicBlock *>, uint32_t>
-        PhiTranslateMap;
-    PhiTranslateMap PhiTranslateTable;
-
     AliasAnalysis *AA;
     MemoryDependenceResults *MD;
     DominatorTree *DT;
@@ -94,10 +79,6 @@ public:
                              Value *LHS, Value *RHS);
     Expression createExtractvalueExpr(ExtractValueInst *EI);
     uint32_t lookupOrAddCall(CallInst *C);
-    uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock,
-                              uint32_t Num, GVN &Gvn);
-    std::pair<uint32_t, bool> assignExpNewValueNum(Expression &exp);
-    bool areAllValsInBB(uint32_t num, const BasicBlock *BB, GVN &Gvn);
 
   public:
     ValueTable();
@@ -106,11 +87,9 @@ public:
     ~ValueTable();
 
     uint32_t lookupOrAdd(Value *V);
-    uint32_t lookup(Value *V, bool Verify = true) const;
+    uint32_t lookup(Value *V) const;
     uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred,
                             Value *LHS, Value *RHS);
-    uint32_t phiTranslate(const BasicBlock *BB, const BasicBlock *PhiBlock,
-                          uint32_t Num, GVN &Gvn);
     bool exists(Value *V) const;
     void add(Value *V, uint32_t num);
     void clear();
@@ -152,10 +131,6 @@ private:
   SmallMapVector<llvm::Value *, llvm::Constant *, 4> ReplaceWithConstMap;
   SmallVector<Instruction *, 8> InstrsToErase;
 
-  // Map the block to reversed postorder traversal number. It is used to
-  // find back edge easily.
-  DenseMap<const BasicBlock *, uint32_t> BlockRPONumber;
-
   typedef SmallVector<NonLocalDepResult, 64> LoadDepVect;
   typedef SmallVector<gvn::AvailableValueInBlock, 64> AvailValInBlkVect;
   typedef SmallVector<BasicBlock *, 64> UnavailBlkVect;
@@ -239,7 +214,7 @@ private:
   bool performPRE(Function &F);
   bool performScalarPRE(Instruction *I);
   bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
-                                 BasicBlock *Curr, unsigned int ValNo);
+                                 unsigned int ValNo);
   Value *findLeader(const BasicBlock *BB, uint32_t num);
   void cleanupGlobalSets();
   void verifyRemoved(const Instruction *I) const;
@@ -251,7 +226,6 @@ private:
   bool processFoldableCondBr(BranchInst *BI);
   void addDeadBlock(BasicBlock *BB);
   void assignValNumForDeadCode();
-  void assignBlockRPONumber(Function &F);
 };
 
 /// Create a legacy GVN pass. This also allows parameterizing whether or not
diff --git a/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
index e4906b709e4bf..4554b5cbc6440 100644
--- a/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
+++ b/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
@@ -17,21 +17,39 @@
 
 namespace llvm {
 
+class ConstantInt;
 class Instruction;
 class MemCpyInst;
 class MemMoveInst;
 class MemSetInst;
+class TargetTransformInfo;
 class Value;
 
 /// Emit a loop implementing the semantics of llvm.memcpy with the equivalent
 /// arguments at \p InsertBefore.
-void createMemCpyLoop(Instruction *InsertBefore,
-                      Value *SrcAddr, Value *DstAddr, Value *CopyLen,
-                      unsigned SrcAlign, unsigned DestAlign,
+void createMemCpyLoop(Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr,
+                      Value *CopyLen, unsigned SrcAlign, unsigned DestAlign,
                       bool SrcIsVolatile, bool DstIsVolatile);
 
+/// Emit a loop implementing the semantics of llvm.memcpy where the size is not
+/// a compile-time constant. Loop will be insterted at \p InsertBefore.
+void createMemCpyLoopUnknownSize(Instruction *InsertBefore, Value *SrcAddr,
+                                 Value *DstAddr, Value *CopyLen,
+                                 unsigned SrcAlign, unsigned DestAlign,
+                                 bool SrcIsVolatile, bool DstIsVolatile,
+                                 const TargetTransformInfo &TTI);
+
+/// Emit a loop implementing the semantics of an llvm.memcpy whose size is a
+/// compile time constant. Loop is inserted at \p InsertBefore.
+void createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr,
+                               Value *DstAddr, ConstantInt *CopyLen,
+                               unsigned SrcAlign, unsigned DestAlign,
+                               bool SrcIsVolatile, bool DstIsVolatile,
+                               const TargetTransformInfo &TTI);
+
+
 /// Expand \p MemCpy as a loop. \p MemCpy is not deleted.
-void expandMemCpyAsLoop(MemCpyInst *MemCpy);
+void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI);
 
 /// Expand \p MemMove as a loop. \p MemMove is not deleted.
 void expandMemMoveAsLoop(MemMoveInst *MemMove);
diff --git a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
index b0448fed9f4d9..2dd205d8b2af2 100644
--- a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
+++ b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
@@ -22,10 +22,10 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Debug.h"
 
-namespace llvm {
-
 #define DEBUG_TYPE "ssaupdater"
 
+namespace llvm {
+
 class CastInst;
 class PHINode;
 template<typename T> class SSAUpdaterTraits;
@@ -453,8 +453,8 @@ public:
   }
 };
 
-#undef DEBUG_TYPE // "ssaupdater"
+} // end llvm namespace
 
-} // End llvm namespace
+#undef DEBUG_TYPE // "ssaupdater"
 
-#endif
+#endif // LLVM_TRANSFORMS_UTILS_SSAUPDATERIMPL_H
diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap
index dd419e861316e..766198bbc5de9 100644
--- a/include/llvm/module.modulemap
+++ b/include/llvm/module.modulemap
@@ -23,6 +23,7 @@ module LLVM_Backend {
     exclude header "CodeGen/CommandFlags.h"
     exclude header "CodeGen/LinkAllAsmWriterComponents.h"
     exclude header "CodeGen/LinkAllCodegenComponents.h"
+    exclude header "CodeGen/GlobalISel/InstructionSelectorImpl.h"
 
     // These are intended for (repeated) textual inclusion.
     textual header "CodeGen/DIEValue.def"