vendor/llvm/llvm-trunk-r351319

73 files changed, 5608 insertions, 2382 deletions

diff --git a/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 2d92b93ca704..0a5908f43790 100644
--- a/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -18,13 +18,14 @@
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
 #include "WebAssembly.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCParser/MCTargetAsmParser.h"
-#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/TargetRegistry.h"
 
@@ -34,27 +35,10 @@ using namespace llvm;
 
 namespace {
 
-// We store register types as SimpleValueType to retain SIMD layout
-// information, but must also be able to supply them as the (unnamed)
-// register enum from WebAssemblyRegisterInfo.td/.inc.
-static unsigned MVTToWasmReg(MVT::SimpleValueType Type) {
-  switch(Type) {
-    case MVT::i32: return WebAssembly::I32_0;
-    case MVT::i64: return WebAssembly::I64_0;
-    case MVT::f32: return WebAssembly::F32_0;
-    case MVT::f64: return WebAssembly::F64_0;
-    case MVT::v16i8: return WebAssembly::V128_0;
-    case MVT::v8i16: return WebAssembly::V128_0;
-    case MVT::v4i32: return WebAssembly::V128_0;
-    case MVT::v4f32: return WebAssembly::V128_0;
-    default: return MVT::INVALID_SIMPLE_VALUE_TYPE;
-  }
-}
-
 /// WebAssemblyOperand - Instances of this class represent the operands in a
 /// parsed WASM machine instruction.
 struct WebAssemblyOperand : public MCParsedAsmOperand {
-  enum KindTy { Token, Local, Stack, Integer, Float, Symbol } Kind;
+  enum KindTy { Token, Integer, Float, Symbol, BrList } Kind;
 
   SMLoc StartLoc, EndLoc;
 
@@ -62,19 +46,6 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
     StringRef Tok;
   };
 
-  struct RegOp {
-    // This is a (virtual) local or stack register represented as 0..
-    unsigned RegNo;
-    // In most targets, the register number also encodes the type, but for
-    // wasm we have to track that seperately since we have an unbounded
-    // number of registers.
-    // This has the unfortunate side effect that we supply a different value
-    // to the table-gen matcher at different times in the process (when it
-    // calls getReg() or addRegOperands().
-    // TODO: While this works, it feels brittle. and would be nice to clean up.
-    MVT::SimpleValueType Type;
-  };
-
   struct IntOp {
     int64_t Val;
   };
@@ -87,37 +58,45 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
     const MCExpr *Exp;
   };
 
+  struct BrLOp {
+    std::vector<unsigned> List;
+  };
+
   union {
     struct TokOp Tok;
-    struct RegOp Reg;
     struct IntOp Int;
     struct FltOp Flt;
     struct SymOp Sym;
+    struct BrLOp BrL;
   };
 
   WebAssemblyOperand(KindTy K, SMLoc Start, SMLoc End, TokOp T)
-    : Kind(K), StartLoc(Start), EndLoc(End), Tok(T) {}
-  WebAssemblyOperand(KindTy K, SMLoc Start, SMLoc End, RegOp R)
-    : Kind(K), StartLoc(Start), EndLoc(End), Reg(R) {}
+      : Kind(K), StartLoc(Start), EndLoc(End), Tok(T) {}
   WebAssemblyOperand(KindTy K, SMLoc Start, SMLoc End, IntOp I)
-    : Kind(K), StartLoc(Start), EndLoc(End), Int(I) {}
+      : Kind(K), StartLoc(Start), EndLoc(End), Int(I) {}
   WebAssemblyOperand(KindTy K, SMLoc Start, SMLoc End, FltOp F)
-    : Kind(K), StartLoc(Start), EndLoc(End), Flt(F) {}
+      : Kind(K), StartLoc(Start), EndLoc(End), Flt(F) {}
   WebAssemblyOperand(KindTy K, SMLoc Start, SMLoc End, SymOp S)
-    : Kind(K), StartLoc(Start), EndLoc(End), Sym(S) {}
+      : Kind(K), StartLoc(Start), EndLoc(End), Sym(S) {}
+  WebAssemblyOperand(KindTy K, SMLoc Start, SMLoc End)
+      : Kind(K), StartLoc(Start), EndLoc(End), BrL() {}
+
+  ~WebAssemblyOperand() {
+    if (isBrList())
+      BrL.~BrLOp();
+  }
 
   bool isToken() const override { return Kind == Token; }
-  bool isImm() const override { return Kind == Integer ||
-                                       Kind == Float ||
-                                       Kind == Symbol; }
-  bool isReg() const override { return Kind == Local || Kind == Stack; }
+  bool isImm() const override {
+    return Kind == Integer || Kind == Float || Kind == Symbol;
+  }
   bool isMem() const override { return false; }
+  bool isReg() const override { return false; }
+  bool isBrList() const { return Kind == BrList; }
 
   unsigned getReg() const override {
-    assert(isReg());
-    // This is called from the tablegen matcher (MatchInstructionImpl)
-    // where it expects to match the type of register, see RegOp above.
-    return MVTToWasmReg(Reg.Type);
+    llvm_unreachable("Assembly inspects a register operand");
+    return 0;
   }
 
   StringRef getToken() const {
@@ -128,19 +107,9 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return StartLoc; }
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  void addRegOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    assert(isReg() && "Not a register operand!");
-    // This is called from the tablegen matcher (MatchInstructionImpl)
-    // where it expects to output the actual register index, see RegOp above.
-    unsigned R = Reg.RegNo;
-    if (Kind == Stack) {
-      // A stack register is represented as a large negative number.
-      // See WebAssemblyRegNumbering::runOnMachineFunction and
-      // getWARegStackId for why this | is needed.
-      R |= INT32_MIN;
-    }
-    Inst.addOperand(MCOperand::createReg(R));
+  void addRegOperands(MCInst &, unsigned) const {
+    // Required by the assembly matcher.
+    llvm_unreachable("Assembly matcher creates register operands");
   }
 
   void addImmOperands(MCInst &Inst, unsigned N) const {
@@ -155,17 +124,17 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
       llvm_unreachable("Should be immediate or symbol!");
   }
 
+  void addBrListOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && isBrList() && "Invalid BrList!");
+    for (auto Br : BrL.List)
+      Inst.addOperand(MCOperand::createImm(Br));
+  }
+
   void print(raw_ostream &OS) const override {
     switch (Kind) {
     case Token:
       OS << "Tok:" << Tok.Tok;
       break;
-    case Local:
-      OS << "Loc:" << Reg.RegNo << ":" << static_cast<int>(Reg.Type);
-      break;
-    case Stack:
-      OS << "Stk:" << Reg.RegNo << ":" << static_cast<int>(Reg.Type);
-      break;
     case Integer:
       OS << "Int:" << Int.Val;
       break;
@@ -175,6 +144,9 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
     case Symbol:
       OS << "Sym:" << Sym.Exp;
       break;
+    case BrList:
+      OS << "BrList:" << BrL.List.size();
+      break;
     }
   }
 };
@@ -182,352 +154,526 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
 class WebAssemblyAsmParser final : public MCTargetAsmParser {
   MCAsmParser &Parser;
   MCAsmLexer &Lexer;
-  // These are for the current function being parsed:
-  // These are vectors since register assignments are so far non-sparse.
-  // Replace by map if necessary.
-  std::vector<MVT::SimpleValueType> LocalTypes;
-  std::vector<MVT::SimpleValueType> StackTypes;
-  MCSymbol *LastLabel;
+
+  // Much like WebAssemblyAsmPrinter in the backend, we have to own these.
+  std::vector<std::unique_ptr<wasm::WasmSignature>> Signatures;
+
+  // Order of labels, directives and instructions in a .s file have no
+  // syntactical enforcement. This class is a callback from the actual parser,
+  // and yet we have to be feeding data to the streamer in a very particular
+  // order to ensure a correct binary encoding that matches the regular backend
+  // (the streamer does not enforce this). This "state machine" enum helps
+  // guarantee that correct order.
+  enum ParserState {
+    FileStart,
+    Label,
+    FunctionStart,
+    FunctionLocals,
+    Instructions,
+  } CurrentState = FileStart;
+
+  // For ensuring blocks are properly nested.
+  enum NestingType {
+    Function,
+    Block,
+    Loop,
+    Try,
+    If,
+    Else,
+    Undefined,
+  };
+  std::vector<NestingType> NestingStack;
+
+  // We track this to see if a .functype following a label is the same,
+  // as this is how we recognize the start of a function.
+  MCSymbol *LastLabel = nullptr;
 
 public:
-  WebAssemblyAsmParser(const MCSubtargetInfo &sti, MCAsmParser &Parser,
-                       const MCInstrInfo &mii, const MCTargetOptions &Options)
-      : MCTargetAsmParser(Options, sti, mii), Parser(Parser),
-        Lexer(Parser.getLexer()), LastLabel(nullptr) {
+  WebAssemblyAsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser,
+                       const MCInstrInfo &MII, const MCTargetOptions &Options)
+      : MCTargetAsmParser(Options, STI, MII), Parser(Parser),
+        Lexer(Parser.getLexer()) {
+    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
 
 #define GET_ASSEMBLER_HEADER
 #include "WebAssemblyGenAsmMatcher.inc"
 
   // TODO: This is required to be implemented, but appears unused.
-  bool ParseRegister(unsigned &/*RegNo*/, SMLoc &/*StartLoc*/,
-                     SMLoc &/*EndLoc*/) override {
+  bool ParseRegister(unsigned & /*RegNo*/, SMLoc & /*StartLoc*/,
+                     SMLoc & /*EndLoc*/) override {
     llvm_unreachable("ParseRegister is not implemented.");
   }
 
-  bool Error(const StringRef &msg, const AsmToken &tok) {
-    return Parser.Error(tok.getLoc(), msg + tok.getString());
+  bool error(const Twine &Msg, const AsmToken &Tok) {
+    return Parser.Error(Tok.getLoc(), Msg + Tok.getString());
+  }
+
+  bool error(const Twine &Msg) {
+    return Parser.Error(Lexer.getTok().getLoc(), Msg);
+  }
+
+  void addSignature(std::unique_ptr<wasm::WasmSignature> &&Sig) {
+    Signatures.push_back(std::move(Sig));
+  }
+
+  std::pair<StringRef, StringRef> nestingString(NestingType NT) {
+    switch (NT) {
+    case Function:
+      return {"function", "end_function"};
+    case Block:
+      return {"block", "end_block"};
+    case Loop:
+      return {"loop", "end_loop"};
+    case Try:
+      return {"try", "end_try"};
+    case If:
+      return {"if", "end_if"};
+    case Else:
+      return {"else", "end_if"};
+    default:
+      llvm_unreachable("unknown NestingType");
+    }
+  }
+
+  void push(NestingType NT) { NestingStack.push_back(NT); }
+
+  bool pop(StringRef Ins, NestingType NT1, NestingType NT2 = Undefined) {
+    if (NestingStack.empty())
+      return error(Twine("End of block construct with no start: ") + Ins);
+    auto Top = NestingStack.back();
+    if (Top != NT1 && Top != NT2)
+      return error(Twine("Block construct type mismatch, expected: ") +
+                   nestingString(Top).second + ", instead got: " + Ins);
+    NestingStack.pop_back();
+    return false;
+  }
+
+  bool ensureEmptyNestingStack() {
+    auto err = !NestingStack.empty();
+    while (!NestingStack.empty()) {
+      error(Twine("Unmatched block construct(s) at function end: ") +
+            nestingString(NestingStack.back()).first);
+      NestingStack.pop_back();
+    }
+    return err;
   }
 
-  bool IsNext(AsmToken::TokenKind Kind) {
-    auto ok = Lexer.is(Kind);
-    if (ok) Parser.Lex();
-    return ok;
+  bool isNext(AsmToken::TokenKind Kind) {
+    auto Ok = Lexer.is(Kind);
+    if (Ok)
+      Parser.Lex();
+    return Ok;
   }
 
-  bool Expect(AsmToken::TokenKind Kind, const char *KindName) {
-    if (!IsNext(Kind))
-      return Error(std::string("Expected ") + KindName + ", instead got: ",
+  bool expect(AsmToken::TokenKind Kind, const char *KindName) {
+    if (!isNext(Kind))
+      return error(std::string("Expected ") + KindName + ", instead got: ",
                    Lexer.getTok());
     return false;
   }
 
-  MVT::SimpleValueType ParseRegType(const StringRef &RegType) {
-    // Derive type from .param .local decls, or the instruction itself.
-    return StringSwitch<MVT::SimpleValueType>(RegType)
-        .Case("i32", MVT::i32)
-        .Case("i64", MVT::i64)
-        .Case("f32", MVT::f32)
-        .Case("f64", MVT::f64)
-        .Case("i8x16", MVT::v16i8)
-        .Case("i16x8", MVT::v8i16)
-        .Case("i32x4", MVT::v4i32)
-        .Case("f32x4", MVT::v4f32)
-        .Default(MVT::INVALID_SIMPLE_VALUE_TYPE);
+  StringRef expectIdent() {
+    if (!Lexer.is(AsmToken::Identifier)) {
+      error("Expected identifier, got: ", Lexer.getTok());
+      return StringRef();
+    }
+    auto Name = Lexer.getTok().getString();
+    Parser.Lex();
+    return Name;
   }
 
-  MVT::SimpleValueType &GetType(
-      std::vector<MVT::SimpleValueType> &Types, size_t i) {
-    Types.resize(std::max(i + 1, Types.size()), MVT::INVALID_SIMPLE_VALUE_TYPE);
-    return Types[i];
+  Optional<wasm::ValType> parseType(const StringRef &Type) {
+    // FIXME: can't use StringSwitch because wasm::ValType doesn't have a
+    // "invalid" value.
+    if (Type == "i32")
+      return wasm::ValType::I32;
+    if (Type == "i64")
+      return wasm::ValType::I64;
+    if (Type == "f32")
+      return wasm::ValType::F32;
+    if (Type == "f64")
+      return wasm::ValType::F64;
+    if (Type == "v128" || Type == "i8x16" || Type == "i16x8" ||
+        Type == "i32x4" || Type == "i64x2" || Type == "f32x4" ||
+        Type == "f64x2")
+      return wasm::ValType::V128;
+    return Optional<wasm::ValType>();
   }
 
-  bool ParseReg(OperandVector &Operands, StringRef TypePrefix) {
-    if (Lexer.is(AsmToken::Integer)) {
-      auto &Local = Lexer.getTok();
-      // This is a reference to a local, turn it into a virtual register.
-      auto LocalNo = static_cast<unsigned>(Local.getIntVal());
-      Operands.push_back(make_unique<WebAssemblyOperand>(
-                           WebAssemblyOperand::Local, Local.getLoc(),
-                           Local.getEndLoc(),
-                           WebAssemblyOperand::RegOp{LocalNo,
-                               GetType(LocalTypes, LocalNo)}));
-      Parser.Lex();
-    } else if (Lexer.is(AsmToken::Identifier)) {
-      auto &StackRegTok = Lexer.getTok();
-      // These are push/pop/drop pseudo stack registers, which we turn
-      // into virtual registers also. The stackify pass will later turn them
-      // back into implicit stack references if possible.
-      auto StackReg = StackRegTok.getString();
-      auto StackOp = StackReg.take_while([](char c) { return isalpha(c); });
-      auto Reg = StackReg.drop_front(StackOp.size());
-      unsigned long long ParsedRegNo = 0;
-      if (!Reg.empty() && getAsUnsignedInteger(Reg, 10, ParsedRegNo))
-        return Error("Cannot parse stack register index: ", StackRegTok);
-      unsigned RegNo = static_cast<unsigned>(ParsedRegNo);
-      if (StackOp == "push") {
-        // This defines a result, record register type.
-        auto RegType = ParseRegType(TypePrefix);
-        GetType(StackTypes, RegNo) = RegType;
-        Operands.push_back(make_unique<WebAssemblyOperand>(
-                             WebAssemblyOperand::Stack,
-                             StackRegTok.getLoc(),
-                             StackRegTok.getEndLoc(),
-                             WebAssemblyOperand::RegOp{RegNo, RegType}));
-      } else if (StackOp == "pop") {
-        // This uses a previously defined stack value.
-        auto RegType = GetType(StackTypes, RegNo);
-        Operands.push_back(make_unique<WebAssemblyOperand>(
-                             WebAssemblyOperand::Stack,
-                             StackRegTok.getLoc(),
-                             StackRegTok.getEndLoc(),
-                             WebAssemblyOperand::RegOp{RegNo, RegType}));
-      } else if (StackOp == "drop") {
-        // This operand will be dropped, since it is part of an instruction
-        // whose result is void.
-      } else {
-        return Error("Unknown stack register prefix: ", StackRegTok);
-      }
+  WebAssembly::ExprType parseBlockType(StringRef ID) {
+    return StringSwitch<WebAssembly::ExprType>(ID)
+        .Case("i32", WebAssembly::ExprType::I32)
+        .Case("i64", WebAssembly::ExprType::I64)
+        .Case("f32", WebAssembly::ExprType::F32)
+        .Case("f64", WebAssembly::ExprType::F64)
+        .Case("v128", WebAssembly::ExprType::V128)
+        .Case("except_ref", WebAssembly::ExprType::ExceptRef)
+        .Case("void", WebAssembly::ExprType::Void)
+        .Default(WebAssembly::ExprType::Invalid);
+  }
+
+  bool parseRegTypeList(SmallVectorImpl<wasm::ValType> &Types) {
+    while (Lexer.is(AsmToken::Identifier)) {
+      auto Type = parseType(Lexer.getTok().getString());
+      if (!Type)
+        return true;
+      Types.push_back(Type.getValue());
       Parser.Lex();
-    } else {
-      return Error(
-            "Expected identifier/integer following $, instead got: ",
-            Lexer.getTok());
+      if (!isNext(AsmToken::Comma))
+        break;
     }
-    IsNext(AsmToken::Equal);
     return false;
   }
 
-  void ParseSingleInteger(bool IsNegative, OperandVector &Operands) {
+  void parseSingleInteger(bool IsNegative, OperandVector &Operands) {
     auto &Int = Lexer.getTok();
     int64_t Val = Int.getIntVal();
-    if (IsNegative) Val = -Val;
+    if (IsNegative)
+      Val = -Val;
     Operands.push_back(make_unique<WebAssemblyOperand>(
-                         WebAssemblyOperand::Integer, Int.getLoc(),
-                         Int.getEndLoc(), WebAssemblyOperand::IntOp{Val}));
+        WebAssemblyOperand::Integer, Int.getLoc(), Int.getEndLoc(),
+        WebAssemblyOperand::IntOp{Val}));
     Parser.Lex();
   }
 
-  bool ParseOperandStartingWithInteger(bool IsNegative,
-                                       OperandVector &Operands,
-                                       StringRef InstType) {
-    ParseSingleInteger(IsNegative, Operands);
-    if (Lexer.is(AsmToken::LParen)) {
-      // Parse load/store operands of the form: offset($reg)align
-      auto &LParen = Lexer.getTok();
-      Operands.push_back(
-            make_unique<WebAssemblyOperand>(WebAssemblyOperand::Token,
-                                            LParen.getLoc(),
-                                            LParen.getEndLoc(),
-                                            WebAssemblyOperand::TokOp{
-                                              LParen.getString()}));
-      Parser.Lex();
-      if (Expect(AsmToken::Dollar, "register")) return true;
-      if (ParseReg(Operands, InstType)) return true;
-      auto &RParen = Lexer.getTok();
-      Operands.push_back(
-            make_unique<WebAssemblyOperand>(WebAssemblyOperand::Token,
-                                            RParen.getLoc(),
-                                            RParen.getEndLoc(),
-                                            WebAssemblyOperand::TokOp{
-                                              RParen.getString()}));
-      if (Expect(AsmToken::RParen, ")")) return true;
-      if (Lexer.is(AsmToken::Integer)) {
-        ParseSingleInteger(false, Operands);
+  bool parseOperandStartingWithInteger(bool IsNegative, OperandVector &Operands,
+                                       StringRef InstName) {
+    parseSingleInteger(IsNegative, Operands);
+    // FIXME: there is probably a cleaner way to do this.
+    auto IsLoadStore = InstName.startswith("load") ||
+                       InstName.startswith("store") ||
+                       InstName.startswith("atomic_load") ||
+                       InstName.startswith("atomic_store");
+    if (IsLoadStore) {
+      // Parse load/store operands of the form: offset align
+      auto &Offset = Lexer.getTok();
+      if (Offset.is(AsmToken::Integer)) {
+        parseSingleInteger(false, Operands);
       } else {
         // Alignment not specified.
         // FIXME: correctly derive a default from the instruction.
+        // We can't just call WebAssembly::GetDefaultP2Align since we don't have
+        // an opcode until after the assembly matcher.
         Operands.push_back(make_unique<WebAssemblyOperand>(
-                             WebAssemblyOperand::Integer, RParen.getLoc(),
-                             RParen.getEndLoc(), WebAssemblyOperand::IntOp{0}));
+            WebAssemblyOperand::Integer, Offset.getLoc(), Offset.getEndLoc(),
+            WebAssemblyOperand::IntOp{0}));
       }
     }
     return false;
   }
 
-  bool ParseInstruction(ParseInstructionInfo &/*Info*/, StringRef Name,
+  void addBlockTypeOperand(OperandVector &Operands, SMLoc NameLoc,
+                           WebAssembly::ExprType BT) {
+    Operands.push_back(make_unique<WebAssemblyOperand>(
+        WebAssemblyOperand::Integer, NameLoc, NameLoc,
+        WebAssemblyOperand::IntOp{static_cast<int64_t>(BT)}));
+  }
+
+  bool ParseInstruction(ParseInstructionInfo & /*Info*/, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override {
-    Operands.push_back(
-          make_unique<WebAssemblyOperand>(WebAssemblyOperand::Token, NameLoc,
-                                          SMLoc::getFromPointer(
-                                            NameLoc.getPointer() + Name.size()),
-                                          WebAssemblyOperand::TokOp{
-                                            StringRef(NameLoc.getPointer(),
-                                                    Name.size())}));
+    // Note: Name does NOT point into the sourcecode, but to a local, so
+    // use NameLoc instead.
+    Name = StringRef(NameLoc.getPointer(), Name.size());
+
+    // WebAssembly has instructions with / in them, which AsmLexer parses
+    // as seperate tokens, so if we find such tokens immediately adjacent (no
+    // whitespace), expand the name to include them:
+    for (;;) {
+      auto &Sep = Lexer.getTok();
+      if (Sep.getLoc().getPointer() != Name.end() ||
+          Sep.getKind() != AsmToken::Slash)
+        break;
+      // Extend name with /
+      Name = StringRef(Name.begin(), Name.size() + Sep.getString().size());
+      Parser.Lex();
+      // We must now find another identifier, or error.
+      auto &Id = Lexer.getTok();
+      if (Id.getKind() != AsmToken::Identifier ||
+          Id.getLoc().getPointer() != Name.end())
+        return error("Incomplete instruction name: ", Id);
+      Name = StringRef(Name.begin(), Name.size() + Id.getString().size());
+      Parser.Lex();
+    }
+
+    // Now construct the name as first operand.
+    Operands.push_back(make_unique<WebAssemblyOperand>(
+        WebAssemblyOperand::Token, NameLoc, SMLoc::getFromPointer(Name.end()),
+        WebAssemblyOperand::TokOp{Name}));
     auto NamePair = Name.split('.');
     // If no '.', there is no type prefix.
-    if (NamePair.second.empty()) std::swap(NamePair.first, NamePair.second);
+    auto BaseName = NamePair.second.empty() ? NamePair.first : NamePair.second;
+
+    // If this instruction is part of a control flow structure, ensure
+    // proper nesting.
+    bool ExpectBlockType = false;
+    if (BaseName == "block") {
+      push(Block);
+      ExpectBlockType = true;
+    } else if (BaseName == "loop") {
+      push(Loop);
+      ExpectBlockType = true;
+    } else if (BaseName == "try") {
+      push(Try);
+      ExpectBlockType = true;
+    } else if (BaseName == "if") {
+      push(If);
+      ExpectBlockType = true;
+    } else if (BaseName == "else") {
+      if (pop(BaseName, If))
+        return true;
+      push(Else);
+    } else if (BaseName == "catch") {
+      if (pop(BaseName, Try))
+        return true;
+      push(Try);
+    } else if (BaseName == "catch_all") {
+      if (pop(BaseName, Try))
+        return true;
+      push(Try);
+    } else if (BaseName == "end_if") {
+      if (pop(BaseName, If, Else))
+        return true;
+    } else if (BaseName == "end_try") {
+      if (pop(BaseName, Try))
+        return true;
+    } else if (BaseName == "end_loop") {
+      if (pop(BaseName, Loop))
+        return true;
+    } else if (BaseName == "end_block") {
+      if (pop(BaseName, Block))
+        return true;
+    } else if (BaseName == "end_function") {
+      if (pop(BaseName, Function) || ensureEmptyNestingStack())
+        return true;
+    }
+
     while (Lexer.isNot(AsmToken::EndOfStatement)) {
       auto &Tok = Lexer.getTok();
       switch (Tok.getKind()) {
-      case AsmToken::Dollar: {
-        Parser.Lex();
-        if (ParseReg(Operands, NamePair.first)) return true;
-        break;
-      }
       case AsmToken::Identifier: {
         auto &Id = Lexer.getTok();
-        const MCExpr *Val;
-        SMLoc End;
-        if (Parser.parsePrimaryExpr(Val, End))
-          return Error("Cannot parse symbol: ", Lexer.getTok());
-        Operands.push_back(make_unique<WebAssemblyOperand>(
-                             WebAssemblyOperand::Symbol, Id.getLoc(),
-                             Id.getEndLoc(), WebAssemblyOperand::SymOp{Val}));
+        if (ExpectBlockType) {
+          // Assume this identifier is a block_type.
+          auto BT = parseBlockType(Id.getString());
+          if (BT == WebAssembly::ExprType::Invalid)
+            return error("Unknown block type: ", Id);
+          addBlockTypeOperand(Operands, NameLoc, BT);
+          Parser.Lex();
+        } else {
+          // Assume this identifier is a label.
+          const MCExpr *Val;
+          SMLoc End;
+          if (Parser.parsePrimaryExpr(Val, End))
+            return error("Cannot parse symbol: ", Lexer.getTok());
+          Operands.push_back(make_unique<WebAssemblyOperand>(
+              WebAssemblyOperand::Symbol, Id.getLoc(), Id.getEndLoc(),
+              WebAssemblyOperand::SymOp{Val}));
+        }
         break;
       }
       case AsmToken::Minus:
         Parser.Lex();
         if (Lexer.isNot(AsmToken::Integer))
-          return Error("Expected integer instead got: ", Lexer.getTok());
-        if (ParseOperandStartingWithInteger(true, Operands, NamePair.first))
+          return error("Expected integer instead got: ", Lexer.getTok());
+        if (parseOperandStartingWithInteger(true, Operands, BaseName))
           return true;
         break;
       case AsmToken::Integer:
-        if (ParseOperandStartingWithInteger(false, Operands, NamePair.first))
+        if (parseOperandStartingWithInteger(false, Operands, BaseName))
           return true;
         break;
       case AsmToken::Real: {
         double Val;
         if (Tok.getString().getAsDouble(Val, false))
-          return Error("Cannot parse real: ", Tok);
+          return error("Cannot parse real: ", Tok);
         Operands.push_back(make_unique<WebAssemblyOperand>(
-                             WebAssemblyOperand::Float, Tok.getLoc(),
-                             Tok.getEndLoc(), WebAssemblyOperand::FltOp{Val}));
+            WebAssemblyOperand::Float, Tok.getLoc(), Tok.getEndLoc(),
+            WebAssemblyOperand::FltOp{Val}));
+        Parser.Lex();
+        break;
+      }
+      case AsmToken::LCurly: {
         Parser.Lex();
+        auto Op = make_unique<WebAssemblyOperand>(
+            WebAssemblyOperand::BrList, Tok.getLoc(), Tok.getEndLoc());
+        if (!Lexer.is(AsmToken::RCurly))
+          for (;;) {
+            Op->BrL.List.push_back(Lexer.getTok().getIntVal());
+            expect(AsmToken::Integer, "integer");
+            if (!isNext(AsmToken::Comma))
+              break;
+          }
+        expect(AsmToken::RCurly, "}");
+        Operands.push_back(std::move(Op));
         break;
       }
       default:
-        return Error("Unexpected token in operand: ", Tok);
+        return error("Unexpected token in operand: ", Tok);
       }
       if (Lexer.isNot(AsmToken::EndOfStatement)) {
-        if (Expect(AsmToken::Comma, ",")) return true;
-      }
-    }
-    Parser.Lex();
-    // Call instructions are vararg, but the tablegen matcher doesn't seem to
-    // support that, so for now we strip these extra operands.
-    // This is problematic if these arguments are not simple $pop stack
-    // registers, since e.g. a local register would get lost, so we check for
-    // this. This can be the case when using -disable-wasm-explicit-locals
-    // which currently s2wasm requires.
-    // TODO: Instead, we can move this code to MatchAndEmitInstruction below and
-    // actually generate get_local instructions on the fly.
-    // Or even better, improve the matcher to support vararg?
-    auto IsIndirect = NamePair.second == "call_indirect";
-    if (IsIndirect || NamePair.second == "call") {
-      // Figure out number of fixed operands from the instruction.
-      size_t CallOperands = 1;  // The name token.
-      if (!IsIndirect) CallOperands++;  // The function index.
-      if (!NamePair.first.empty()) CallOperands++;  // The result register.
-      if (Operands.size() > CallOperands) {
-        // Ensure operands we drop are all $pop.
-        for (size_t I = CallOperands; I < Operands.size(); I++) {
-          auto Operand =
-              reinterpret_cast<WebAssemblyOperand *>(Operands[I].get());
-          if (Operand->Kind != WebAssemblyOperand::Stack)
-            Parser.Error(NameLoc,
-              "Call instruction has non-stack arguments, if this code was "
-              "generated with -disable-wasm-explicit-locals please remove it");
-        }
-        // Drop unneeded operands.
-        Operands.resize(CallOperands);
+        if (expect(AsmToken::Comma, ","))
+          return true;
       }
     }
-    // Block instructions require a signature index, but these are missing in
-    // assembly, so we add a dummy one explicitly (since we have no control
-    // over signature tables here, we assume these will be regenerated when
-    // the wasm module is generated).
-    if (NamePair.second == "block" || NamePair.second == "loop") {
-      Operands.push_back(make_unique<WebAssemblyOperand>(
-                           WebAssemblyOperand::Integer, NameLoc,
-                           NameLoc, WebAssemblyOperand::IntOp{-1}));
-    }
-    // These don't specify the type, which has to derived from the local index.
-    if (NamePair.second == "get_local" || NamePair.second == "tee_local") {
-      if (Operands.size() >= 3 && Operands[1]->isReg() &&
-          Operands[2]->isImm()) {
-        auto Op1 = reinterpret_cast<WebAssemblyOperand *>(Operands[1].get());
-        auto Op2 = reinterpret_cast<WebAssemblyOperand *>(Operands[2].get());
-        auto Type = GetType(LocalTypes, static_cast<size_t>(Op2->Int.Val));
-        Op1->Reg.Type = Type;
-        GetType(StackTypes, Op1->Reg.RegNo) = Type;
-      }
+    if (ExpectBlockType && Operands.size() == 1) {
+      // Support blocks with no operands as default to void.
+      addBlockTypeOperand(Operands, NameLoc, WebAssembly::ExprType::Void);
     }
+    Parser.Lex();
     return false;
   }
 
   void onLabelParsed(MCSymbol *Symbol) override {
     LastLabel = Symbol;
+    CurrentState = Label;
   }
 
+  bool parseSignature(wasm::WasmSignature *Signature) {
+    if (expect(AsmToken::LParen, "("))
+      return true;
+    if (parseRegTypeList(Signature->Params))
+      return true;
+    if (expect(AsmToken::RParen, ")"))
+      return true;
+    if (expect(AsmToken::MinusGreater, "->"))
+      return true;
+    if (expect(AsmToken::LParen, "("))
+      return true;
+    if (parseRegTypeList(Signature->Returns))
+      return true;
+    if (expect(AsmToken::RParen, ")"))
+      return true;
+    return false;
+  }
+
+  // This function processes wasm-specific directives streamed to
+  // WebAssemblyTargetStreamer, all others go to the generic parser
+  // (see WasmAsmParser).
   bool ParseDirective(AsmToken DirectiveID) override {
+    // This function has a really weird return value behavior that is different
+    // from all the other parsing functions:
+    // - return true && no tokens consumed -> don't know this directive / let
+    //   the generic parser handle it.
+    // - return true && tokens consumed -> a parsing error occurred.
+    // - return false -> processed this directive successfully.
     assert(DirectiveID.getKind() == AsmToken::Identifier);
     auto &Out = getStreamer();
-    auto &TOut = reinterpret_cast<WebAssemblyTargetStreamer &>(
-                   *Out.getTargetStreamer());
-    // TODO: we're just parsing the subset of directives we're interested in,
-    // and ignoring ones we don't recognise. We should ideally verify
-    // all directives here.
-    if (DirectiveID.getString() == ".type") {
-      // This could be the start of a function, check if followed by
-      // "label,@function"
-      if (!(IsNext(AsmToken::Identifier) &&
-            IsNext(AsmToken::Comma) &&
-            IsNext(AsmToken::At) &&
-            Lexer.is(AsmToken::Identifier)))
-        return Error("Expected label,@type declaration, got: ", Lexer.getTok());
-      if (Lexer.getTok().getString() == "function") {
-        // Track locals from start of function.
-        LocalTypes.clear();
-        StackTypes.clear();
-      }
-      Parser.Lex();
-      //Out.EmitSymbolAttribute(??, MCSA_ELF_TypeFunction);
-    } else if (DirectiveID.getString() == ".param" ||
-               DirectiveID.getString() == ".local") {
-      // Track the number of locals, needed for correct virtual register
-      // assignment elsewhere.
-      // Also output a directive to the streamer.
-      std::vector<MVT> Params;
-      std::vector<MVT> Locals;
-      while (Lexer.is(AsmToken::Identifier)) {
-        auto RegType = ParseRegType(Lexer.getTok().getString());
-        if (RegType == MVT::INVALID_SIMPLE_VALUE_TYPE) return true;
-        LocalTypes.push_back(RegType);
-        if (DirectiveID.getString() == ".param") {
-          Params.push_back(RegType);
-        } else {
-          Locals.push_back(RegType);
-        }
-        Parser.Lex();
-        if (!IsNext(AsmToken::Comma)) break;
+    auto &TOut =
+        reinterpret_cast<WebAssemblyTargetStreamer &>(*Out.getTargetStreamer());
+
+    // TODO: any time we return an error, at least one token must have been
+    // consumed, otherwise this will not signal an error to the caller.
+    if (DirectiveID.getString() == ".globaltype") {
+      auto SymName = expectIdent();
+      if (SymName.empty())
+        return true;
+      if (expect(AsmToken::Comma, ","))
+        return true;
+      auto TypeTok = Lexer.getTok();
+      auto TypeName = expectIdent();
+      if (TypeName.empty())
+        return true;
+      auto Type = parseType(TypeName);
+      if (!Type)
+        return error("Unknown type in .globaltype directive: ", TypeTok);
+      // Now set this symbol with the correct type.
+      auto WasmSym = cast<MCSymbolWasm>(
+          TOut.getStreamer().getContext().getOrCreateSymbol(SymName));
+      WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
+      WasmSym->setGlobalType(
+          wasm::WasmGlobalType{uint8_t(Type.getValue()), true});
+      // And emit the directive again.
+      TOut.emitGlobalType(WasmSym);
+      return expect(AsmToken::EndOfStatement, "EOL");
+    }
+
+    if (DirectiveID.getString() == ".functype") {
+      // This code has to send things to the streamer similar to
+      // WebAssemblyAsmPrinter::EmitFunctionBodyStart.
+      // TODO: would be good to factor this into a common function, but the
+      // assembler and backend really don't share any common code, and this code
+      // parses the locals seperately.
+      auto SymName = expectIdent();
+      if (SymName.empty())
+        return true;
+      auto WasmSym = cast<MCSymbolWasm>(
+          TOut.getStreamer().getContext().getOrCreateSymbol(SymName));
+      if (CurrentState == Label && WasmSym == LastLabel) {
+        // This .functype indicates a start of a function.
+        if (ensureEmptyNestingStack())
+          return true;
+        CurrentState = FunctionStart;
+        push(Function);
       }
-      assert(LastLabel);
-      TOut.emitParam(LastLabel, Params);
+      auto Signature = make_unique<wasm::WasmSignature>();
+      if (parseSignature(Signature.get()))
+        return true;
+      WasmSym->setSignature(Signature.get());
+      addSignature(std::move(Signature));
+      WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
+      TOut.emitFunctionType(WasmSym);
+      // TODO: backend also calls TOut.emitIndIdx, but that is not implemented.
+      return expect(AsmToken::EndOfStatement, "EOL");
+    }
+
+    if (DirectiveID.getString() == ".eventtype") {
+      auto SymName = expectIdent();
+      if (SymName.empty())
+        return true;
+      auto WasmSym = cast<MCSymbolWasm>(
+          TOut.getStreamer().getContext().getOrCreateSymbol(SymName));
+      auto Signature = make_unique<wasm::WasmSignature>();
+      if (parseRegTypeList(Signature->Params))
+        return true;
+      WasmSym->setSignature(Signature.get());
+      addSignature(std::move(Signature));
+      WasmSym->setType(wasm::WASM_SYMBOL_TYPE_EVENT);
+      TOut.emitEventType(WasmSym);
+      // TODO: backend also calls TOut.emitIndIdx, but that is not implemented.
+      return expect(AsmToken::EndOfStatement, "EOL");
+    }
+
+    if (DirectiveID.getString() == ".local") {
+      if (CurrentState != FunctionStart)
+        return error(".local directive should follow the start of a function",
+                     Lexer.getTok());
+      SmallVector<wasm::ValType, 4> Locals;
+      if (parseRegTypeList(Locals))
+        return true;
       TOut.emitLocal(Locals);
-    } else {
-      // For now, ignore anydirective we don't recognize:
-      while (Lexer.isNot(AsmToken::EndOfStatement)) Parser.Lex();
+      CurrentState = FunctionLocals;
+      return expect(AsmToken::EndOfStatement, "EOL");
     }
-    return Expect(AsmToken::EndOfStatement, "EOL");
+
+    return true; // We didn't process this directive.
   }
 
-  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &/*Opcode*/,
-                               OperandVector &Operands,
-                               MCStreamer &Out, uint64_t &ErrorInfo,
+  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned & /*Opcode*/,
+                               OperandVector &Operands, MCStreamer &Out,
+                               uint64_t &ErrorInfo,
                                bool MatchingInlineAsm) override {
     MCInst Inst;
     unsigned MatchResult =
         MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm);
     switch (MatchResult) {
     case Match_Success: {
+      if (CurrentState == FunctionStart) {
+        // This is the first instruction in a function, but we haven't seen
+        // a .local directive yet. The streamer requires locals to be encoded
+        // as a prelude to the instructions, so emit an empty list of locals
+        // here.
+        auto &TOut = reinterpret_cast<WebAssemblyTargetStreamer &>(
+            *Out.getTargetStreamer());
+        TOut.emitLocal(SmallVector<wasm::ValType, 0>());
+      }
+      CurrentState = Instructions;
       Out.EmitInstruction(Inst, getSTI());
       return false;
     }
     case Match_MissingFeature:
-      return Parser.Error(IDLoc,
-          "instruction requires a WASM feature not currently enabled");
+      return Parser.Error(
+          IDLoc, "instruction requires a WASM feature not currently enabled");
     case Match_MnemonicFail:
       return Parser.Error(IDLoc, "invalid instruction");
     case Match_NearMisses:
@@ -547,6 +693,8 @@ public:
     }
     llvm_unreachable("Implement any new match types added!");
   }
+
+  void onEndOfFile() override { ensureEmptyNestingStack(); }
 };
 } // end anonymous namespace
 
diff --git a/lib/Target/WebAssembly/CMakeLists.txt b/lib/Target/WebAssembly/CMakeLists.txt
index a928f110efe0..1f3b7d942d47 100644
--- a/lib/Target/WebAssembly/CMakeLists.txt
+++ b/lib/Target/WebAssembly/CMakeLists.txt
@@ -19,7 +19,9 @@ add_llvm_target(WebAssemblyCodeGen
   WebAssemblyCallIndirectFixup.cpp
   WebAssemblyCFGStackify.cpp
   WebAssemblyCFGSort.cpp
+  WebAssemblyDebugValueManager.cpp
   WebAssemblyLateEHPrepare.cpp
+  WebAssemblyEHRestoreStackPointer.cpp
   WebAssemblyExceptionInfo.cpp
   WebAssemblyExplicitLocals.cpp
   WebAssemblyFastISel.cpp
@@ -46,7 +48,7 @@ add_llvm_target(WebAssemblyCodeGen
   WebAssemblyRuntimeLibcallSignatures.cpp
   WebAssemblySelectionDAGInfo.cpp
   WebAssemblySetP2AlignOperands.cpp
-  WebAssemblyStoreResults.cpp
+  WebAssemblyMemIntrinsicResults.cpp
   WebAssemblySubtarget.cpp
   WebAssemblyTargetMachine.cpp
   WebAssemblyTargetObjectFile.cpp
diff --git a/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 2f0960271e30..6acc9b20eed2 100644
--- a/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -16,7 +16,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
-#include "WebAssembly.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCFixedLenDisassembler.h"
@@ -37,6 +36,8 @@ using DecodeStatus = MCDisassembler::DecodeStatus;
 #include "WebAssemblyGenDisassemblerTables.inc"
 
 namespace {
+static constexpr int WebAssemblyInstructionTableSize = 256;
+
 class WebAssemblyDisassembler final : public MCDisassembler {
   std::unique_ptr<const MCInstrInfo> MCII;
 
@@ -75,31 +76,43 @@ static int nextByte(ArrayRef<uint8_t> Bytes, uint64_t &Size) {
   return V;
 }
 
-static bool parseLEBImmediate(MCInst &MI, uint64_t &Size,
-                              ArrayRef<uint8_t> Bytes, bool Signed) {
+static bool nextLEB(int64_t &Val, ArrayRef<uint8_t> Bytes, uint64_t &Size,
+                    bool Signed = false) {
   unsigned N = 0;
   const char *Error = nullptr;
-  auto Val = Signed ? decodeSLEB128(Bytes.data() + Size, &N,
-                                    Bytes.data() + Bytes.size(), &Error)
-                    : static_cast<int64_t>(
-                          decodeULEB128(Bytes.data() + Size, &N,
-                                        Bytes.data() + Bytes.size(), &Error));
+  Val = Signed ? decodeSLEB128(Bytes.data() + Size, &N,
+                               Bytes.data() + Bytes.size(), &Error)
+               : static_cast<int64_t>(decodeULEB128(Bytes.data() + Size, &N,
+                                                    Bytes.data() + Bytes.size(),
+                                                    &Error));
   if (Error)
     return false;
   Size += N;
+  return true;
+}
+
+static bool parseLEBImmediate(MCInst &MI, uint64_t &Size,
+                              ArrayRef<uint8_t> Bytes, bool Signed) {
+  int64_t Val;
+  if (!nextLEB(Val, Bytes, Size, Signed))
+    return false;
   MI.addOperand(MCOperand::createImm(Val));
   return true;
 }
 
 template <typename T>
-bool parseFPImmediate(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes) {
+bool parseImmediate(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes) {
   if (Size + sizeof(T) > Bytes.size())
     return false;
   T Val;
   memcpy(&Val, Bytes.data() + Size, sizeof(T));
   support::endian::byte_swap<T, support::endianness::little>(Val);
   Size += sizeof(T);
-  MI.addOperand(MCOperand::createFPImm(static_cast<double>(Val)));
+  if (std::is_floating_point<T>::value) {
+    MI.addOperand(MCOperand::createFPImm(static_cast<double>(Val)));
+  } else {
+    MI.addOperand(MCOperand::createImm(static_cast<int64_t>(Val)));
+  }
   return true;
 }
 
@@ -108,7 +121,7 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
     raw_ostream & /*OS*/, raw_ostream &CS) const {
   CommentStream = &CS;
   Size = 0;
-  auto Opc = nextByte(Bytes, Size);
+  int Opc = nextByte(Bytes, Size);
   if (Opc < 0)
     return MCDisassembler::Fail;
   const auto *WasmInst = &InstructionTable0[Opc];
@@ -124,10 +137,12 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
     }
     if (!WasmInst)
       return MCDisassembler::Fail;
-    Opc = nextByte(Bytes, Size);
-    if (Opc < 0)
+    int64_t PrefixedOpc;
+    if (!nextLEB(PrefixedOpc, Bytes, Size))
       return MCDisassembler::Fail;
-    WasmInst += Opc;
+    if (PrefixedOpc < 0 || PrefixedOpc >= WebAssemblyInstructionTableSize)
+      return MCDisassembler::Fail;
+    WasmInst += PrefixedOpc;
   }
   if (WasmInst->ET == ET_Unused)
     return MCDisassembler::Fail;
@@ -136,7 +151,8 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
   MI.setOpcode(WasmInst->Opcode);
   // Parse any operands.
   for (uint8_t OPI = 0; OPI < WasmInst->NumOperands; OPI++) {
-    switch (WasmInst->Operands[OPI]) {
+    auto OT = OperandTable[WasmInst->OperandStart + OPI];
+    switch (OT) {
     // ULEB operands:
     case WebAssembly::OPERAND_BASIC_BLOCK:
     case WebAssembly::OPERAND_LOCAL:
@@ -152,32 +168,68 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
     }
     // SLEB operands:
     case WebAssembly::OPERAND_I32IMM:
-    case WebAssembly::OPERAND_I64IMM:
-    case WebAssembly::OPERAND_SIGNATURE: {
+    case WebAssembly::OPERAND_I64IMM: {
       if (!parseLEBImmediate(MI, Size, Bytes, true))
         return MCDisassembler::Fail;
       break;
     }
+    // block_type operands (uint8_t).
+    case WebAssembly::OPERAND_SIGNATURE: {
+      if (!parseImmediate<uint8_t>(MI, Size, Bytes))
+        return MCDisassembler::Fail;
+      break;
+    }
     // FP operands.
     case WebAssembly::OPERAND_F32IMM: {
-      if (!parseFPImmediate<float>(MI, Size, Bytes))
+      if (!parseImmediate<float>(MI, Size, Bytes))
         return MCDisassembler::Fail;
       break;
     }
     case WebAssembly::OPERAND_F64IMM: {
-      if (!parseFPImmediate<double>(MI, Size, Bytes))
+      if (!parseImmediate<double>(MI, Size, Bytes))
         return MCDisassembler::Fail;
       break;
     }
-    case MCOI::OPERAND_REGISTER: {
-      // These are NOT actually in the instruction stream, but MC is going to
-      // expect operands to be present for them!
-      // FIXME: can MC re-generate register assignments or do we have to
-      // do this? Since this function decodes a single instruction, we don't
-      // have the proper context for tracking an operand stack here.
-      MI.addOperand(MCOperand::createReg(0));
+    // Vector lane operands (not LEB encoded).
+    case WebAssembly::OPERAND_VEC_I8IMM: {
+      if (!parseImmediate<uint8_t>(MI, Size, Bytes))
+        return MCDisassembler::Fail;
+      break;
+    }
+    case WebAssembly::OPERAND_VEC_I16IMM: {
+      if (!parseImmediate<uint16_t>(MI, Size, Bytes))
+        return MCDisassembler::Fail;
+      break;
+    }
+    case WebAssembly::OPERAND_VEC_I32IMM: {
+      if (!parseImmediate<uint32_t>(MI, Size, Bytes))
+        return MCDisassembler::Fail;
+      break;
+    }
+    case WebAssembly::OPERAND_VEC_I64IMM: {
+      if (!parseImmediate<uint64_t>(MI, Size, Bytes))
+        return MCDisassembler::Fail;
+      break;
+    }
+    case WebAssembly::OPERAND_BRLIST: {
+      int64_t TargetTableLen;
+      if (!nextLEB(TargetTableLen, Bytes, Size, false))
+        return MCDisassembler::Fail;
+      for (int64_t I = 0; I < TargetTableLen; I++) {
+        if (!parseLEBImmediate(MI, Size, Bytes, false))
+          return MCDisassembler::Fail;
+      }
+      // Default case.
+      if (!parseLEBImmediate(MI, Size, Bytes, false))
+        return MCDisassembler::Fail;
       break;
     }
+    case MCOI::OPERAND_REGISTER:
+      // The tablegen header currently does not have any register operands since
+      // we use only the stack (_S) instructions.
+      // If you hit this that probably means a bad instruction definition in
+      // tablegen.
+      llvm_unreachable("Register operand in WebAssemblyDisassembler");
     default:
       llvm_unreachable("Unknown operand type in WebAssemblyDisassembler");
     }
diff --git a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
index 10fa798ac8d7..15532d7ff1a6 100644
--- a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
+++ b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
@@ -35,12 +35,12 @@ using namespace llvm;
 WebAssemblyInstPrinter::WebAssemblyInstPrinter(const MCAsmInfo &MAI,
                                                const MCInstrInfo &MII,
                                                const MCRegisterInfo &MRI)
-    : MCInstPrinter(MAI, MII, MRI), ControlFlowCounter(0) {}
+    : MCInstPrinter(MAI, MII, MRI) {}
 
 void WebAssemblyInstPrinter::printRegName(raw_ostream &OS,
                                           unsigned RegNo) const {
   assert(RegNo != WebAssemblyFunctionInfo::UnusedReg);
-  // Note that there's an implicit get_local/set_local here!
+  // Note that there's an implicit local.get/local.set here!
   OS << "$" << RegNo;
 }
 
@@ -57,9 +57,9 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
       // FIXME: For CALL_INDIRECT_VOID, don't print a leading comma, because
       // we have an extra flags operand which is not currently printed, for
       // compatiblity reasons.
-      if (i != 0 &&
-          (MI->getOpcode() != WebAssembly::CALL_INDIRECT_VOID ||
-           i != Desc.getNumOperands()))
+      if (i != 0 && ((MI->getOpcode() != WebAssembly::CALL_INDIRECT_VOID &&
+                      MI->getOpcode() != WebAssembly::CALL_INDIRECT_VOID_S) ||
+                     i != Desc.getNumOperands()))
         OS << ", ";
       printOperand(MI, i, OS);
     }
@@ -70,25 +70,76 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
   if (CommentStream) {
     // Observe any effects on the control flow stack, for use in annotating
     // control flow label references.
-    switch (MI->getOpcode()) {
+    unsigned Opc = MI->getOpcode();
+    switch (Opc) {
     default:
       break;
-    case WebAssembly::LOOP: {
+
+    case WebAssembly::LOOP:
+    case WebAssembly::LOOP_S:
       printAnnotation(OS, "label" + utostr(ControlFlowCounter) + ':');
       ControlFlowStack.push_back(std::make_pair(ControlFlowCounter++, true));
       break;
-    }
+
     case WebAssembly::BLOCK:
+    case WebAssembly::BLOCK_S:
       ControlFlowStack.push_back(std::make_pair(ControlFlowCounter++, false));
       break;
+
+    case WebAssembly::TRY:
+    case WebAssembly::TRY_S:
+      ControlFlowStack.push_back(std::make_pair(ControlFlowCounter++, false));
+      EHPadStack.push_back(EHPadStackCounter++);
+      LastSeenEHInst = TRY;
+      break;
+
     case WebAssembly::END_LOOP:
-      // Have to guard against an empty stack, in case of mismatched pairs
-      // in assembly parsing.
-      if (!ControlFlowStack.empty()) ControlFlowStack.pop_back();
+    case WebAssembly::END_LOOP_S:
+      if (ControlFlowStack.empty()) {
+        printAnnotation(OS, "End marker mismatch!");
+      } else {
+        ControlFlowStack.pop_back();
+      }
       break;
+
     case WebAssembly::END_BLOCK:
-      if (!ControlFlowStack.empty()) printAnnotation(
-          OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
+    case WebAssembly::END_BLOCK_S:
+      if (ControlFlowStack.empty()) {
+        printAnnotation(OS, "End marker mismatch!");
+      } else {
+        printAnnotation(
+            OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
+      }
+      break;
+
+    case WebAssembly::END_TRY:
+    case WebAssembly::END_TRY_S:
+      if (ControlFlowStack.empty()) {
+        printAnnotation(OS, "End marker mismatch!");
+      } else {
+        printAnnotation(
+            OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
+        LastSeenEHInst = END_TRY;
+      }
+      break;
+
+    case WebAssembly::CATCH_I32:
+    case WebAssembly::CATCH_I32_S:
+    case WebAssembly::CATCH_I64:
+    case WebAssembly::CATCH_I64_S:
+    case WebAssembly::CATCH_ALL:
+    case WebAssembly::CATCH_ALL_S:
+      // There can be multiple catch instructions for one try instruction, so we
+      // print a label only for the first 'catch' label.
+      if (LastSeenEHInst != CATCH) {
+        if (EHPadStack.empty()) {
+          printAnnotation(OS, "try-catch mismatch!");
+        } else {
+          printAnnotation(OS,
+                          "catch" + utostr(EHPadStack.pop_back_val()) + ':');
+        }
+      }
+      LastSeenEHInst = CATCH;
       break;
     }
 
@@ -96,34 +147,61 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
     unsigned NumFixedOperands = Desc.NumOperands;
     SmallSet<uint64_t, 8> Printed;
     for (unsigned i = 0, e = MI->getNumOperands(); i < e; ++i) {
-      if (!(i < NumFixedOperands
-                ? (Desc.OpInfo[i].OperandType ==
-                   WebAssembly::OPERAND_BASIC_BLOCK)
-                : (Desc.TSFlags & WebAssemblyII::VariableOpImmediateIsLabel)))
-        continue;
+      // See if this operand denotes a basic block target.
+      if (i < NumFixedOperands) {
+        // A non-variable_ops operand, check its type.
+        if (Desc.OpInfo[i].OperandType != WebAssembly::OPERAND_BASIC_BLOCK)
+          continue;
+      } else {
+        // A variable_ops operand, which currently can be immediates (used in
+        // br_table) which are basic block targets, or for call instructions
+        // when using -wasm-keep-registers (in which case they are registers,
+        // and should not be processed).
+        if (!MI->getOperand(i).isImm())
+          continue;
+      }
       uint64_t Depth = MI->getOperand(i).getImm();
       if (!Printed.insert(Depth).second)
         continue;
-      const auto &Pair = ControlFlowStack.rbegin()[Depth];
-      printAnnotation(OS, utostr(Depth) + ": " + (Pair.second ? "up" : "down") +
-                              " to label" + utostr(Pair.first));
+
+      if (Opc == WebAssembly::RETHROW || Opc == WebAssembly::RETHROW_S) {
+        if (Depth > EHPadStack.size()) {
+          printAnnotation(OS, "Invalid depth argument!");
+        } else if (Depth == EHPadStack.size()) {
+          // This can happen when rethrow instruction breaks out of all nests
+          // and throws up to the current function's caller.
+          printAnnotation(OS, utostr(Depth) + ": " + "to caller");
+        } else {
+          uint64_t CatchNo = EHPadStack.rbegin()[Depth];
+          printAnnotation(OS, utostr(Depth) + ": " + "down to catch" +
+                                  utostr(CatchNo));
+        }
+
+      } else {
+        if (Depth >= ControlFlowStack.size()) {
+          printAnnotation(OS, "Invalid depth argument!");
+        } else {
+          const auto &Pair = ControlFlowStack.rbegin()[Depth];
+          printAnnotation(OS, utostr(Depth) + ": " +
+                                  (Pair.second ? "up" : "down") + " to label" +
+                                  utostr(Pair.first));
+        }
+      }
     }
   }
 }
 
 static std::string toString(const APFloat &FP) {
   // Print NaNs with custom payloads specially.
-  if (FP.isNaN() &&
-      !FP.bitwiseIsEqual(APFloat::getQNaN(FP.getSemantics())) &&
+  if (FP.isNaN() && !FP.bitwiseIsEqual(APFloat::getQNaN(FP.getSemantics())) &&
       !FP.bitwiseIsEqual(
           APFloat::getQNaN(FP.getSemantics(), /*Negative=*/true))) {
     APInt AI = FP.bitcastToAPInt();
-    return
-        std::string(AI.isNegative() ? "-" : "") + "nan:0x" +
-        utohexstr(AI.getZExtValue() &
-                  (AI.getBitWidth() == 32 ? INT64_C(0x007fffff) :
-                                            INT64_C(0x000fffffffffffff)),
-                  /*LowerCase=*/true);
+    return std::string(AI.isNegative() ? "-" : "") + "nan:0x" +
+           utohexstr(AI.getZExtValue() &
+                         (AI.getBitWidth() == 32 ? INT64_C(0x007fffff)
+                                                 : INT64_C(0x000fffffffffffff)),
+                     /*LowerCase=*/true);
   }
 
   // Use C99's hexadecimal floating-point representation.
@@ -141,9 +219,6 @@ void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                           raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
   if (Op.isReg()) {
-    assert((OpNo < MII.get(MI->getOpcode()).getNumOperands() ||
-            MII.get(MI->getOpcode()).TSFlags == 0) &&
-           "WebAssembly variable_ops register ops don't use TSFlags");
     unsigned WAReg = Op.getReg();
     if (int(WAReg) >= 0)
       printRegName(O, WAReg);
@@ -157,23 +232,9 @@ void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     if (OpNo < MII.get(MI->getOpcode()).getNumDefs())
       O << '=';
   } else if (Op.isImm()) {
-    const MCInstrDesc &Desc = MII.get(MI->getOpcode());
-    assert((OpNo < Desc.getNumOperands() ||
-            (Desc.TSFlags & WebAssemblyII::VariableOpIsImmediate)) &&
-           "WebAssemblyII::VariableOpIsImmediate should be set for "
-           "variable_ops immediate ops");
-    (void)Desc;
-    // TODO: (MII.get(MI->getOpcode()).TSFlags &
-    //        WebAssemblyII::VariableOpImmediateIsLabel)
-    // can tell us whether this is an immediate referencing a label in the
-    // control flow stack, and it may be nice to pretty-print.
     O << Op.getImm();
   } else if (Op.isFPImm()) {
     const MCInstrDesc &Desc = MII.get(MI->getOpcode());
-    assert(OpNo < Desc.getNumOperands() &&
-           "Unexpected floating-point immediate as a non-fixed operand");
-    assert(Desc.TSFlags == 0 &&
-           "WebAssembly variable_ops floating point ops don't use TSFlags");
     const MCOperandInfo &Info = Desc.OpInfo[OpNo];
     if (Info.OperandType == WebAssembly::OPERAND_F32IMM) {
       // TODO: MC converts all floating point immediate operands to double.
@@ -184,78 +245,66 @@ void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
       O << ::toString(APFloat(Op.getFPImm()));
     }
   } else {
-    assert((OpNo < MII.get(MI->getOpcode()).getNumOperands() ||
-            (MII.get(MI->getOpcode()).TSFlags &
-             WebAssemblyII::VariableOpIsImmediate)) &&
-           "WebAssemblyII::VariableOpIsImmediate should be set for "
-           "variable_ops expr ops");
     assert(Op.isExpr() && "unknown operand kind in printOperand");
     Op.getExpr()->print(O, &MAI);
   }
 }
 
-void WebAssemblyInstPrinter::printWebAssemblyP2AlignOperand(
-    const MCInst *MI, unsigned OpNo, raw_ostream &O) {
+void WebAssemblyInstPrinter::printBrList(const MCInst *MI, unsigned OpNo,
+                                         raw_ostream &O) {
+  O << "{";
+  for (unsigned I = OpNo, E = MI->getNumOperands(); I != E; ++I) {
+    if (I != OpNo)
+      O << ", ";
+    O << MI->getOperand(I).getImm();
+  }
+  O << "}";
+}
+
+void WebAssemblyInstPrinter::printWebAssemblyP2AlignOperand(const MCInst *MI,
+                                                            unsigned OpNo,
+                                                            raw_ostream &O) {
   int64_t Imm = MI->getOperand(OpNo).getImm();
   if (Imm == WebAssembly::GetDefaultP2Align(MI->getOpcode()))
     return;
   O << ":p2align=" << Imm;
 }
 
-void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(
-    const MCInst *MI, unsigned OpNo, raw_ostream &O) {
-  int64_t Imm = MI->getOperand(OpNo).getImm();
-  switch (WebAssembly::ExprType(Imm)) {
-  case WebAssembly::ExprType::Void: break;
-  case WebAssembly::ExprType::I32: O << "i32"; break;
-  case WebAssembly::ExprType::I64: O << "i64"; break;
-  case WebAssembly::ExprType::F32: O << "f32"; break;
-  case WebAssembly::ExprType::F64: O << "f64"; break;
-  case WebAssembly::ExprType::I8x16: O << "i8x16"; break;
-  case WebAssembly::ExprType::I16x8: O << "i16x8"; break;
-  case WebAssembly::ExprType::I32x4: O << "i32x4"; break;
-  case WebAssembly::ExprType::F32x4: O << "f32x4"; break;
-  case WebAssembly::ExprType::B8x16: O << "b8x16"; break;
-  case WebAssembly::ExprType::B16x8: O << "b16x8"; break;
-  case WebAssembly::ExprType::B32x4: O << "b32x4"; break;
-  case WebAssembly::ExprType::ExceptRef: O << "except_ref"; break;
-  }
+void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(const MCInst *MI,
+                                                              unsigned OpNo,
+                                                              raw_ostream &O) {
+  auto Imm = static_cast<unsigned>(MI->getOperand(OpNo).getImm());
+  if (Imm != wasm::WASM_TYPE_NORESULT)
+    O << WebAssembly::anyTypeToString(Imm);
 }
 
-const char *llvm::WebAssembly::TypeToString(MVT Ty) {
-  switch (Ty.SimpleTy) {
-  case MVT::i32:
+// We have various enums representing a subset of these types, use this
+// function to convert any of them to text.
+const char *llvm::WebAssembly::anyTypeToString(unsigned Ty) {
+  switch (Ty) {
+  case wasm::WASM_TYPE_I32:
     return "i32";
-  case MVT::i64:
+  case wasm::WASM_TYPE_I64:
     return "i64";
-  case MVT::f32:
+  case wasm::WASM_TYPE_F32:
     return "f32";
-  case MVT::f64:
+  case wasm::WASM_TYPE_F64:
     return "f64";
-  case MVT::v16i8:
-  case MVT::v8i16:
-  case MVT::v4i32:
-  case MVT::v4f32:
+  case wasm::WASM_TYPE_V128:
     return "v128";
-  case MVT::ExceptRef:
+  case wasm::WASM_TYPE_FUNCREF:
+    return "funcref";
+  case wasm::WASM_TYPE_FUNC:
+    return "func";
+  case wasm::WASM_TYPE_EXCEPT_REF:
     return "except_ref";
+  case wasm::WASM_TYPE_NORESULT:
+    return "void";
   default:
-    llvm_unreachable("unsupported type");
+    return "invalid_type";
   }
 }
 
-const char *llvm::WebAssembly::TypeToString(wasm::ValType Type) {
-  switch (Type) {
-  case wasm::ValType::I32:
-    return "i32";
-  case wasm::ValType::I64:
-    return "i64";
-  case wasm::ValType::F32:
-    return "f32";
-  case wasm::ValType::F64:
-    return "f64";
-  case wasm::ValType::EXCEPT_REF:
-    return "except_ref";
-  }
-  llvm_unreachable("unsupported type");
+const char *llvm::WebAssembly::typeToString(wasm::ValType Ty) {
+  return anyTypeToString(static_cast<unsigned>(Ty));
 }
diff --git a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
index f5b890a7615e..5ad45c7d5c7f 100644
--- a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
+++ b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
@@ -25,8 +25,13 @@ namespace llvm {
 class MCSubtargetInfo;
 
 class WebAssemblyInstPrinter final : public MCInstPrinter {
-  uint64_t ControlFlowCounter;
-  SmallVector<std::pair<uint64_t, bool>, 0> ControlFlowStack;
+  uint64_t ControlFlowCounter = 0;
+  uint64_t EHPadStackCounter = 0;
+  SmallVector<std::pair<uint64_t, bool>, 4> ControlFlowStack;
+  SmallVector<uint64_t, 4> EHPadStack;
+
+  enum EHInstKind { TRY, CATCH, END_TRY };
+  EHInstKind LastSeenEHInst = END_TRY;
 
 public:
   WebAssemblyInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
@@ -38,6 +43,7 @@ public:
 
   // Used by tblegen code.
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printBrList(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printWebAssemblyP2AlignOperand(const MCInst *MI, unsigned OpNo,
                                       raw_ostream &O);
   void printWebAssemblySignatureOperand(const MCInst *MI, unsigned OpNo,
@@ -50,8 +56,8 @@ public:
 
 namespace WebAssembly {
 
-const char *TypeToString(MVT Ty);
-const char *TypeToString(wasm::ValType Type);
+const char *typeToString(wasm::ValType Ty);
+const char *anyTypeToString(unsigned Ty);
 
 } // end namespace WebAssembly
 
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 244c2189b455..0726dd481174 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -73,13 +73,13 @@ public:
 const MCFixupKindInfo &
 WebAssemblyAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[WebAssembly::NumTargetFixupKinds] = {
-    // This table *must* be in the order that the fixup_* kinds are defined in
-    // WebAssemblyFixupKinds.h.
-    //
-    // Name                     Offset (bits) Size (bits)     Flags
-    { "fixup_code_sleb128_i32", 0,            5*8,            0 },
-    { "fixup_code_sleb128_i64", 0,            10*8,           0 },
-    { "fixup_code_uleb128_i32", 0,            5*8,            0 },
+      // This table *must* be in the order that the fixup_* kinds are defined in
+      // WebAssemblyFixupKinds.h.
+      //
+      // Name                     Offset (bits) Size (bits)     Flags
+      {"fixup_code_sleb128_i32", 0, 5 * 8, 0},
+      {"fixup_code_sleb128_i64", 0, 10 * 8, 0},
+      {"fixup_code_uleb128_i32", 0, 5 * 8, 0},
   };
 
   if (Kind < FirstTargetFixupKind)
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
index b0af63c924bd..c2fac5f93a2f 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
@@ -15,11 +15,9 @@
 namespace llvm {
 namespace WebAssembly {
 enum Fixups {
-  fixup_code_sleb128_i32 = FirstTargetFixupKind,      // 32-bit signed
-  fixup_code_sleb128_i64,                             // 64-bit signed
-  fixup_code_uleb128_i32,                             // 32-bit unsigned
-
-  fixup_code_global_index,                            // 32-bit unsigned
+  fixup_code_sleb128_i32 = FirstTargetFixupKind, // 32-bit signed
+  fixup_code_sleb128_i64,                        // 64-bit signed
+  fixup_code_uleb128_i32,                        // 32-bit unsigned
 
   // Marker
   LastTargetFixupKind,
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index 94ca94e1e18c..065a4dc94ca6 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -67,13 +67,16 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
     OS << uint8_t(Binary);
   } else {
     assert(Binary <= UINT16_MAX && "Several-byte opcodes not supported yet");
-    OS << uint8_t(Binary >> 8)
-       << uint8_t(Binary);
+    OS << uint8_t(Binary >> 8);
+    encodeULEB128(uint8_t(Binary), OS);
   }
 
   // For br_table instructions, encode the size of the table. In the MCInst,
-  // there's an index operand, one operand for each table entry, and the
-  // default operand.
+  // there's an index operand (if not a stack instruction), one operand for
+  // each table entry, and the default operand.
+  if (MI.getOpcode() == WebAssembly::BR_TABLE_I32_S ||
+      MI.getOpcode() == WebAssembly::BR_TABLE_I64_S)
+    encodeULEB128(MI.getNumOperands() - 1, OS);
   if (MI.getOpcode() == WebAssembly::BR_TABLE_I32 ||
       MI.getOpcode() == WebAssembly::BR_TABLE_I64)
     encodeULEB128(MI.getNumOperands() - 2, OS);
@@ -83,36 +86,47 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
     const MCOperand &MO = MI.getOperand(i);
     if (MO.isReg()) {
       /* nothing to encode */
+
     } else if (MO.isImm()) {
       if (i < Desc.getNumOperands()) {
-        assert(Desc.TSFlags == 0 &&
-               "WebAssembly non-variable_ops don't use TSFlags");
         const MCOperandInfo &Info = Desc.OpInfo[i];
         LLVM_DEBUG(dbgs() << "Encoding immediate: type="
                           << int(Info.OperandType) << "\n");
-        if (Info.OperandType == WebAssembly::OPERAND_I32IMM) {
+        switch (Info.OperandType) {
+        case WebAssembly::OPERAND_I32IMM:
           encodeSLEB128(int32_t(MO.getImm()), OS);
-        } else if (Info.OperandType == WebAssembly::OPERAND_OFFSET32) {
+          break;
+        case WebAssembly::OPERAND_OFFSET32:
           encodeULEB128(uint32_t(MO.getImm()), OS);
-        } else if (Info.OperandType == WebAssembly::OPERAND_I64IMM) {
+          break;
+        case WebAssembly::OPERAND_I64IMM:
           encodeSLEB128(int64_t(MO.getImm()), OS);
-        } else if (Info.OperandType == WebAssembly::OPERAND_GLOBAL) {
-          llvm_unreachable("wasm globals should only be accessed symbolicly");
-        } else if (Info.OperandType == WebAssembly::OPERAND_SIGNATURE) {
+          break;
+        case WebAssembly::OPERAND_SIGNATURE:
           OS << uint8_t(MO.getImm());
-        } else {
+          break;
+        case WebAssembly::OPERAND_VEC_I8IMM:
+          support::endian::write<uint8_t>(OS, MO.getImm(), support::little);
+          break;
+        case WebAssembly::OPERAND_VEC_I16IMM:
+          support::endian::write<uint16_t>(OS, MO.getImm(), support::little);
+          break;
+        case WebAssembly::OPERAND_VEC_I32IMM:
+          support::endian::write<uint32_t>(OS, MO.getImm(), support::little);
+          break;
+        case WebAssembly::OPERAND_VEC_I64IMM:
+          support::endian::write<uint64_t>(OS, MO.getImm(), support::little);
+          break;
+        case WebAssembly::OPERAND_GLOBAL:
+          llvm_unreachable("wasm globals should only be accessed symbolicly");
+        default:
           encodeULEB128(uint64_t(MO.getImm()), OS);
         }
       } else {
-        assert(Desc.TSFlags == (WebAssemblyII::VariableOpIsImmediate |
-                                WebAssemblyII::VariableOpImmediateIsLabel));
         encodeULEB128(uint64_t(MO.getImm()), OS);
       }
+
     } else if (MO.isFPImm()) {
-      assert(i < Desc.getNumOperands() &&
-             "Unexpected floating-point immediate as a non-fixed operand");
-      assert(Desc.TSFlags == 0 &&
-             "WebAssembly variable_ops floating point ops don't use TSFlags");
       const MCOperandInfo &Info = Desc.OpInfo[i];
       if (Info.OperandType == WebAssembly::OPERAND_F32IMM) {
         // TODO: MC converts all floating point immediate operands to double.
@@ -124,27 +138,31 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
         double d = MO.getFPImm();
         support::endian::write<double>(OS, d, support::little);
       }
+
     } else if (MO.isExpr()) {
       const MCOperandInfo &Info = Desc.OpInfo[i];
       llvm::MCFixupKind FixupKind;
       size_t PaddedSize = 5;
-      if (Info.OperandType == WebAssembly::OPERAND_I32IMM) {
+      switch (Info.OperandType) {
+      case WebAssembly::OPERAND_I32IMM:
         FixupKind = MCFixupKind(WebAssembly::fixup_code_sleb128_i32);
-      } else if (Info.OperandType == WebAssembly::OPERAND_I64IMM) {
+        break;
+      case WebAssembly::OPERAND_I64IMM:
         FixupKind = MCFixupKind(WebAssembly::fixup_code_sleb128_i64);
         PaddedSize = 10;
-      } else if (Info.OperandType == WebAssembly::OPERAND_FUNCTION32 ||
-                 Info.OperandType == WebAssembly::OPERAND_OFFSET32 ||
-                 Info.OperandType == WebAssembly::OPERAND_TYPEINDEX) {
+        break;
+      case WebAssembly::OPERAND_FUNCTION32:
+      case WebAssembly::OPERAND_OFFSET32:
+      case WebAssembly::OPERAND_TYPEINDEX:
+      case WebAssembly::OPERAND_GLOBAL:
+      case WebAssembly::OPERAND_EVENT:
         FixupKind = MCFixupKind(WebAssembly::fixup_code_uleb128_i32);
-      } else if (Info.OperandType == WebAssembly::OPERAND_GLOBAL) {
-        FixupKind = MCFixupKind(WebAssembly::fixup_code_global_index);
-      } else {
+        break;
+      default:
         llvm_unreachable("unexpected symbolic operand kind");
       }
-      Fixups.push_back(MCFixup::create(
-          OS.tell() - Start, MO.getExpr(),
-          FixupKind, MI.getLoc()));
+      Fixups.push_back(MCFixup::create(OS.tell() - Start, MO.getExpr(),
+                                       FixupKind, MI.getLoc()));
       ++MCNumFixups;
       encodeULEB128(0, OS, PaddedSize);
     } else {
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
index baf8a0c96c0a..390f367c2978 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -90,6 +90,10 @@ static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
   return new WebAssemblyTargetAsmStreamer(S, OS);
 }
 
+static MCTargetStreamer *createNullTargetStreamer(MCStreamer &S) {
+  return new WebAssemblyTargetNullStreamer(S);
+}
+
 // Force static initialization.
 extern "C" void LLVMInitializeWebAssemblyTargetMC() {
   for (Target *T :
@@ -120,16 +124,31 @@ extern "C" void LLVMInitializeWebAssemblyTargetMC() {
                                                  createObjectTargetStreamer);
     // Register the asm target streamer.
     TargetRegistry::RegisterAsmTargetStreamer(*T, createAsmTargetStreamer);
+    // Register the null target streamer.
+    TargetRegistry::RegisterNullTargetStreamer(*T, createNullTargetStreamer);
   }
 }
 
 wasm::ValType WebAssembly::toValType(const MVT &Ty) {
   switch (Ty.SimpleTy) {
-  case MVT::i32: return wasm::ValType::I32;
-  case MVT::i64: return wasm::ValType::I64;
-  case MVT::f32: return wasm::ValType::F32;
-  case MVT::f64: return wasm::ValType::F64;
-  case MVT::ExceptRef: return wasm::ValType::EXCEPT_REF;
-  default: llvm_unreachable("unexpected type");
+  case MVT::i32:
+    return wasm::ValType::I32;
+  case MVT::i64:
+    return wasm::ValType::I64;
+  case MVT::f32:
+    return wasm::ValType::F32;
+  case MVT::f64:
+    return wasm::ValType::F64;
+  case MVT::v16i8:
+  case MVT::v8i16:
+  case MVT::v4i32:
+  case MVT::v2i64:
+  case MVT::v4f32:
+  case MVT::v2f64:
+    return wasm::ValType::V128;
+  case MVT::ExceptRef:
+    return wasm::ValType::EXCEPT_REF;
+  default:
+    llvm_unreachable("unexpected type");
   }
 }
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index c1c8d243e920..a01517fb90c3 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -59,6 +59,14 @@ enum OperandType {
   OPERAND_F32IMM,
   /// 64-bit floating-point immediates.
   OPERAND_F64IMM,
+  /// 8-bit vector lane immediate
+  OPERAND_VEC_I8IMM,
+  /// 16-bit vector lane immediate
+  OPERAND_VEC_I16IMM,
+  /// 32-bit vector lane immediate
+  OPERAND_VEC_I32IMM,
+  /// 64-bit vector lane immediate
+  OPERAND_VEC_I64IMM,
   /// 32-bit unsigned function indices.
   OPERAND_FUNCTION32,
   /// 32-bit unsigned memory offsets.
@@ -69,17 +77,24 @@ enum OperandType {
   OPERAND_SIGNATURE,
   /// type signature immediate for call_indirect.
   OPERAND_TYPEINDEX,
+  /// Event index.
+  OPERAND_EVENT,
+  /// A list of branch targets for br_list.
+  OPERAND_BRLIST,
 };
 } // end namespace WebAssembly
 
 namespace WebAssemblyII {
-enum {
-  // For variadic instructions, this flag indicates whether an operand
-  // in the variable_ops range is an immediate value.
-  VariableOpIsImmediate = (1 << 0),
-  // For immediate values in the variable_ops range, this flag indicates
-  // whether the value represents a control-flow label.
-  VariableOpImmediateIsLabel = (1 << 1)
+
+/// Target Operand Flag enum.
+enum TOF {
+  MO_NO_FLAG = 0,
+
+  // Flags to indicate the type of the symbol being referenced
+  MO_SYMBOL_FUNCTION = 0x1,
+  MO_SYMBOL_GLOBAL = 0x2,
+  MO_SYMBOL_EVENT = 0x4,
+  MO_SYMBOL_MASK = 0x7,
 };
 } // end namespace WebAssemblyII
 
@@ -149,6 +164,10 @@ inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_RMW8_U_XCHG_I32_S:
   case WebAssembly::ATOMIC_RMW8_U_XCHG_I64:
   case WebAssembly::ATOMIC_RMW8_U_XCHG_I64_S:
+  case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I32:
+  case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I32_S:
+  case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I64:
+  case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I64_S:
     return 0;
   case WebAssembly::LOAD16_S_I32:
   case WebAssembly::LOAD16_S_I32_S:
@@ -194,6 +213,10 @@ inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_RMW16_U_XCHG_I32_S:
   case WebAssembly::ATOMIC_RMW16_U_XCHG_I64:
   case WebAssembly::ATOMIC_RMW16_U_XCHG_I64_S:
+  case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I32:
+  case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I32_S:
+  case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I64:
+  case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I64_S:
     return 1;
   case WebAssembly::LOAD_I32:
   case WebAssembly::LOAD_I32_S:
@@ -241,6 +264,14 @@ inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_RMW_XCHG_I32_S:
   case WebAssembly::ATOMIC_RMW32_U_XCHG_I64:
   case WebAssembly::ATOMIC_RMW32_U_XCHG_I64_S:
+  case WebAssembly::ATOMIC_RMW_CMPXCHG_I32:
+  case WebAssembly::ATOMIC_RMW_CMPXCHG_I32_S:
+  case WebAssembly::ATOMIC_RMW32_U_CMPXCHG_I64:
+  case WebAssembly::ATOMIC_RMW32_U_CMPXCHG_I64_S:
+  case WebAssembly::ATOMIC_NOTIFY:
+  case WebAssembly::ATOMIC_NOTIFY_S:
+  case WebAssembly::ATOMIC_WAIT_I32:
+  case WebAssembly::ATOMIC_WAIT_I32_S:
     return 2;
   case WebAssembly::LOAD_I64:
   case WebAssembly::LOAD_I64_S:
@@ -266,7 +297,36 @@ inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_RMW_XOR_I64_S:
   case WebAssembly::ATOMIC_RMW_XCHG_I64:
   case WebAssembly::ATOMIC_RMW_XCHG_I64_S:
+  case WebAssembly::ATOMIC_RMW_CMPXCHG_I64:
+  case WebAssembly::ATOMIC_RMW_CMPXCHG_I64_S:
+  case WebAssembly::ATOMIC_WAIT_I64:
+  case WebAssembly::ATOMIC_WAIT_I64_S:
     return 3;
+  case WebAssembly::LOAD_v16i8:
+  case WebAssembly::LOAD_v16i8_S:
+  case WebAssembly::LOAD_v8i16:
+  case WebAssembly::LOAD_v8i16_S:
+  case WebAssembly::LOAD_v4i32:
+  case WebAssembly::LOAD_v4i32_S:
+  case WebAssembly::LOAD_v2i64:
+  case WebAssembly::LOAD_v2i64_S:
+  case WebAssembly::LOAD_v4f32:
+  case WebAssembly::LOAD_v4f32_S:
+  case WebAssembly::LOAD_v2f64:
+  case WebAssembly::LOAD_v2f64_S:
+  case WebAssembly::STORE_v16i8:
+  case WebAssembly::STORE_v16i8_S:
+  case WebAssembly::STORE_v8i16:
+  case WebAssembly::STORE_v8i16_S:
+  case WebAssembly::STORE_v4i32:
+  case WebAssembly::STORE_v4i32_S:
+  case WebAssembly::STORE_v2i64:
+  case WebAssembly::STORE_v2i64_S:
+  case WebAssembly::STORE_v4f32:
+  case WebAssembly::STORE_v4f32_S:
+  case WebAssembly::STORE_v2f64:
+  case WebAssembly::STORE_v2f64_S:
+    return 4;
   default:
     llvm_unreachable("Only loads and stores have p2align values");
   }
@@ -282,19 +342,14 @@ static const unsigned StoreP2AlignOperandNo = 0;
 
 /// This is used to indicate block signatures.
 enum class ExprType : unsigned {
-  Void      = 0x40,
-  I32       = 0x7F,
-  I64       = 0x7E,
-  F32       = 0x7D,
-  F64       = 0x7C,
-  I8x16     = 0x7B,
-  I16x8     = 0x7A,
-  I32x4     = 0x79,
-  F32x4     = 0x78,
-  B8x16     = 0x77,
-  B16x8     = 0x76,
-  B32x4     = 0x75,
-  ExceptRef = 0x68
+  Void = 0x40,
+  I32 = 0x7F,
+  I64 = 0x7E,
+  F32 = 0x7D,
+  F64 = 0x7C,
+  V128 = 0x7B,
+  ExceptRef = 0x68,
+  Invalid = 0x00
 };
 
 /// Instruction opcodes emitted via means other than CodeGen.
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
index 5272e188e1d0..50143fb0ece3 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
@@ -39,70 +39,80 @@ WebAssemblyTargetAsmStreamer::WebAssemblyTargetAsmStreamer(
 WebAssemblyTargetWasmStreamer::WebAssemblyTargetWasmStreamer(MCStreamer &S)
     : WebAssemblyTargetStreamer(S) {}
 
-static void PrintTypes(formatted_raw_ostream &OS, ArrayRef<MVT> Types) {
+static void printTypes(formatted_raw_ostream &OS,
+                       ArrayRef<wasm::ValType> Types) {
   bool First = true;
-  for (MVT Type : Types) {
+  for (auto Type : Types) {
     if (First)
       First = false;
     else
       OS << ", ";
-    OS << WebAssembly::TypeToString(Type);
+    OS << WebAssembly::typeToString(Type);
   }
   OS << '\n';
 }
 
-void WebAssemblyTargetAsmStreamer::emitParam(MCSymbol *Symbol,
-                                             ArrayRef<MVT> Types) {
+void WebAssemblyTargetAsmStreamer::emitLocal(ArrayRef<wasm::ValType> Types) {
   if (!Types.empty()) {
-    OS << "\t.param  \t";
-
-    // FIXME: Currently this applies to the "current" function; it may
-    // be cleaner to specify an explicit symbol as part of the directive.
-
-    PrintTypes(OS, Types);
+    OS << "\t.local  \t";
+    printTypes(OS, Types);
   }
 }
 
-void WebAssemblyTargetAsmStreamer::emitResult(MCSymbol *Symbol,
-                                              ArrayRef<MVT> Types) {
-  if (!Types.empty()) {
-    OS << "\t.result \t";
+void WebAssemblyTargetAsmStreamer::emitEndFunc() { OS << "\t.endfunc\n"; }
 
-    // FIXME: Currently this applies to the "current" function; it may
-    // be cleaner to specify an explicit symbol as part of the directive.
+void WebAssemblyTargetAsmStreamer::emitSignature(
+    const wasm::WasmSignature *Sig) {
+  OS << "(";
+  emitParamList(Sig);
+  OS << ") -> (";
+  emitReturnList(Sig);
+  OS << ")";
+}
 
-    PrintTypes(OS, Types);
+void WebAssemblyTargetAsmStreamer::emitParamList(
+    const wasm::WasmSignature *Sig) {
+  auto &Params = Sig->Params;
+  for (auto &Ty : Params) {
+    if (&Ty != &Params[0])
+      OS << ", ";
+    OS << WebAssembly::typeToString(Ty);
   }
 }
 
-void WebAssemblyTargetAsmStreamer::emitLocal(ArrayRef<MVT> Types) {
-  if (!Types.empty()) {
-    OS << "\t.local  \t";
-    PrintTypes(OS, Types);
+void WebAssemblyTargetAsmStreamer::emitReturnList(
+    const wasm::WasmSignature *Sig) {
+  auto &Returns = Sig->Returns;
+  for (auto &Ty : Returns) {
+    if (&Ty != &Returns[0])
+      OS << ", ";
+    OS << WebAssembly::typeToString(Ty);
   }
 }
 
-void WebAssemblyTargetAsmStreamer::emitEndFunc() { OS << "\t.endfunc\n"; }
+void WebAssemblyTargetAsmStreamer::emitFunctionType(const MCSymbolWasm *Sym) {
+  assert(Sym->isFunction());
+  OS << "\t.functype\t" << Sym->getName() << " ";
+  emitSignature(Sym->getSignature());
+  OS << "\n";
+}
 
-void WebAssemblyTargetAsmStreamer::emitIndirectFunctionType(
-    MCSymbol *Symbol, SmallVectorImpl<MVT> &Params, SmallVectorImpl<MVT> &Results) {
-  OS << "\t.functype\t" << Symbol->getName();
-  if (Results.empty())
-    OS << ", void";
-  else {
-    assert(Results.size() == 1);
-    OS << ", " << WebAssembly::TypeToString(Results.front());
-  }
-  for (auto Ty : Params)
-    OS << ", " << WebAssembly::TypeToString(Ty);
-  OS << '\n';
+void WebAssemblyTargetAsmStreamer::emitGlobalType(const MCSymbolWasm *Sym) {
+  assert(Sym->isGlobal());
+  OS << "\t.globaltype\t" << Sym->getName() << ", "
+     << WebAssembly::typeToString(
+            static_cast<wasm::ValType>(Sym->getGlobalType().Type))
+     << '\n';
 }
 
-void WebAssemblyTargetAsmStreamer::emitGlobalImport(StringRef name) {
-  OS << "\t.import_global\t" << name << '\n';
+void WebAssemblyTargetAsmStreamer::emitEventType(const MCSymbolWasm *Sym) {
+  assert(Sym->isEvent());
+  OS << "\t.eventtype\t" << Sym->getName() << " ";
+  emitParamList(Sym->getSignature());
+  OS << "\n";
 }
 
-void WebAssemblyTargetAsmStreamer::emitImportModule(MCSymbolWasm *Sym,
+void WebAssemblyTargetAsmStreamer::emitImportModule(const MCSymbolWasm *Sym,
                                                     StringRef ModuleName) {
   OS << "\t.import_module\t" << Sym->getName() << ", " << ModuleName << '\n';
 }
@@ -111,27 +121,9 @@ void WebAssemblyTargetAsmStreamer::emitIndIdx(const MCExpr *Value) {
   OS << "\t.indidx  \t" << *Value << '\n';
 }
 
-void WebAssemblyTargetWasmStreamer::emitParam(MCSymbol *Symbol,
-                                              ArrayRef<MVT> Types) {
-  SmallVector<wasm::ValType, 4> Params;
-  for (MVT Ty : Types)
-    Params.push_back(WebAssembly::toValType(Ty));
-
-  cast<MCSymbolWasm>(Symbol)->setParams(std::move(Params));
-}
-
-void WebAssemblyTargetWasmStreamer::emitResult(MCSymbol *Symbol,
-                                               ArrayRef<MVT> Types) {
-  SmallVector<wasm::ValType, 4> Returns;
-  for (MVT Ty : Types)
-    Returns.push_back(WebAssembly::toValType(Ty));
-
-  cast<MCSymbolWasm>(Symbol)->setReturns(std::move(Returns));
-}
-
-void WebAssemblyTargetWasmStreamer::emitLocal(ArrayRef<MVT> Types) {
-  SmallVector<std::pair<MVT, uint32_t>, 4> Grouped;
-  for (MVT Type : Types) {
+void WebAssemblyTargetWasmStreamer::emitLocal(ArrayRef<wasm::ValType> Types) {
+  SmallVector<std::pair<wasm::ValType, uint32_t>, 4> Grouped;
+  for (auto Type : Types) {
     if (Grouped.empty() || Grouped.back().first != Type)
       Grouped.push_back(std::make_pair(Type, 1));
     else
@@ -141,7 +133,7 @@ void WebAssemblyTargetWasmStreamer::emitLocal(ArrayRef<MVT> Types) {
   Streamer.EmitULEB128IntValue(Grouped.size());
   for (auto Pair : Grouped) {
     Streamer.EmitULEB128IntValue(Pair.second);
-    emitValueType(WebAssembly::toValType(Pair.first));
+    emitValueType(Pair.first);
   }
 }
 
@@ -152,34 +144,3 @@ void WebAssemblyTargetWasmStreamer::emitEndFunc() {
 void WebAssemblyTargetWasmStreamer::emitIndIdx(const MCExpr *Value) {
   llvm_unreachable(".indidx encoding not yet implemented");
 }
-
-void WebAssemblyTargetWasmStreamer::emitIndirectFunctionType(
-    MCSymbol *Symbol, SmallVectorImpl<MVT> &Params,
-    SmallVectorImpl<MVT> &Results) {
-  MCSymbolWasm *WasmSym = cast<MCSymbolWasm>(Symbol);
-  if (WasmSym->isFunction()) {
-    // Symbol already has its arguments and result set.
-    return;
-  }
-
-  SmallVector<wasm::ValType, 4> ValParams;
-  for (MVT Ty : Params)
-    ValParams.push_back(WebAssembly::toValType(Ty));
-
-  SmallVector<wasm::ValType, 1> ValResults;
-  for (MVT Ty : Results)
-    ValResults.push_back(WebAssembly::toValType(Ty));
-
-  WasmSym->setParams(std::move(ValParams));
-  WasmSym->setReturns(std::move(ValResults));
-  WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
-}
-
-void WebAssemblyTargetWasmStreamer::emitGlobalImport(StringRef name) {
-  llvm_unreachable(".global_import is not needed for direct wasm output");
-}
-
-void WebAssemblyTargetWasmStreamer::emitImportModule(MCSymbolWasm *Sym,
-                                                     StringRef ModuleName) {
-  Sym->setModuleName(ModuleName);
-}
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
index cafcb04ccd11..3073938118b4 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
@@ -31,24 +31,21 @@ class WebAssemblyTargetStreamer : public MCTargetStreamer {
 public:
   explicit WebAssemblyTargetStreamer(MCStreamer &S);
 
-  /// .param
-  virtual void emitParam(MCSymbol *Symbol, ArrayRef<MVT> Types) = 0;
-  /// .result
-  virtual void emitResult(MCSymbol *Symbol, ArrayRef<MVT> Types) = 0;
   /// .local
-  virtual void emitLocal(ArrayRef<MVT> Types) = 0;
+  virtual void emitLocal(ArrayRef<wasm::ValType> Types) = 0;
   /// .endfunc
   virtual void emitEndFunc() = 0;
   /// .functype
-  virtual void emitIndirectFunctionType(MCSymbol *Symbol,
-                                        SmallVectorImpl<MVT> &Params,
-                                        SmallVectorImpl<MVT> &Results) = 0;
+  virtual void emitFunctionType(const MCSymbolWasm *Sym) = 0;
   /// .indidx
   virtual void emitIndIdx(const MCExpr *Value) = 0;
-  /// .import_global
-  virtual void emitGlobalImport(StringRef name) = 0;
+  /// .globaltype
+  virtual void emitGlobalType(const MCSymbolWasm *Sym) = 0;
+  /// .eventtype
+  virtual void emitEventType(const MCSymbolWasm *Sym) = 0;
   /// .import_module
-  virtual void emitImportModule(MCSymbolWasm *Sym, StringRef ModuleName) = 0;
+  virtual void emitImportModule(const MCSymbolWasm *Sym,
+                                StringRef ModuleName) = 0;
 
 protected:
   void emitValueType(wasm::ValType Type);
@@ -57,20 +54,20 @@ protected:
 /// This part is for ascii assembly output
 class WebAssemblyTargetAsmStreamer final : public WebAssemblyTargetStreamer {
   formatted_raw_ostream &OS;
+  void emitSignature(const wasm::WasmSignature *Sig);
+  void emitParamList(const wasm::WasmSignature *Sig);
+  void emitReturnList(const wasm::WasmSignature *Sig);
 
 public:
   WebAssemblyTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS);
 
-  void emitParam(MCSymbol *Symbol, ArrayRef<MVT> Types) override;
-  void emitResult(MCSymbol *Symbol, ArrayRef<MVT> Types) override;
-  void emitLocal(ArrayRef<MVT> Types) override;
+  void emitLocal(ArrayRef<wasm::ValType> Types) override;
   void emitEndFunc() override;
-  void emitIndirectFunctionType(MCSymbol *Symbol,
-                                SmallVectorImpl<MVT> &Params,
-                                SmallVectorImpl<MVT> &Results) override;
+  void emitFunctionType(const MCSymbolWasm *Sym) override;
   void emitIndIdx(const MCExpr *Value) override;
-  void emitGlobalImport(StringRef name) override;
-  void emitImportModule(MCSymbolWasm *Sym, StringRef ModuleName) override;
+  void emitGlobalType(const MCSymbolWasm *Sym) override;
+  void emitEventType(const MCSymbolWasm *Sym) override;
+  void emitImportModule(const MCSymbolWasm *Sym, StringRef ModuleName) override;
 };
 
 /// This part is for Wasm object output
@@ -78,16 +75,29 @@ class WebAssemblyTargetWasmStreamer final : public WebAssemblyTargetStreamer {
 public:
   explicit WebAssemblyTargetWasmStreamer(MCStreamer &S);
 
-  void emitParam(MCSymbol *Symbol, ArrayRef<MVT> Types) override;
-  void emitResult(MCSymbol *Symbol, ArrayRef<MVT> Types) override;
-  void emitLocal(ArrayRef<MVT> Types) override;
+  void emitLocal(ArrayRef<wasm::ValType> Types) override;
   void emitEndFunc() override;
-  void emitIndirectFunctionType(MCSymbol *Symbol,
-                                SmallVectorImpl<MVT> &Params,
-                                SmallVectorImpl<MVT> &Results) override;
+  void emitFunctionType(const MCSymbolWasm *Sym) override {}
   void emitIndIdx(const MCExpr *Value) override;
-  void emitGlobalImport(StringRef name) override;
-  void emitImportModule(MCSymbolWasm *Sym, StringRef ModuleName) override;
+  void emitGlobalType(const MCSymbolWasm *Sym) override {}
+  void emitEventType(const MCSymbolWasm *Sym) override {}
+  void emitImportModule(const MCSymbolWasm *Sym,
+                        StringRef ModuleName) override {}
+};
+
+/// This part is for null output
+class WebAssemblyTargetNullStreamer final : public WebAssemblyTargetStreamer {
+public:
+  explicit WebAssemblyTargetNullStreamer(MCStreamer &S)
+      : WebAssemblyTargetStreamer(S) {}
+
+  void emitLocal(ArrayRef<wasm::ValType>) override {}
+  void emitEndFunc() override {}
+  void emitFunctionType(const MCSymbolWasm *) override {}
+  void emitIndIdx(const MCExpr *) override {}
+  void emitGlobalType(const MCSymbolWasm *) override {}
+  void emitEventType(const MCSymbolWasm *) override {}
+  void emitImportModule(const MCSymbolWasm *, StringRef) override {}
 };
 
 } // end namespace llvm
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
index 4fb12d40b01b..763e30be8e02 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
@@ -81,16 +81,23 @@ static const MCSection *GetFixupSection(const MCExpr *Expr) {
   return nullptr;
 }
 
-unsigned
-WebAssemblyWasmObjectWriter::getRelocType(const MCValue &Target,
-                                          const MCFixup &Fixup) const {
+static bool IsGlobalType(const MCValue &Target) {
+  const MCSymbolRefExpr *RefA = Target.getSymA();
+  return RefA && RefA->getKind() == MCSymbolRefExpr::VK_WebAssembly_GLOBAL;
+}
+
+static bool IsEventType(const MCValue &Target) {
+  const MCSymbolRefExpr *RefA = Target.getSymA();
+  return RefA && RefA->getKind() == MCSymbolRefExpr::VK_WebAssembly_EVENT;
+}
+
+unsigned WebAssemblyWasmObjectWriter::getRelocType(const MCValue &Target,
+                                                   const MCFixup &Fixup) const {
   // WebAssembly functions are not allocated in the data address space. To
   // resolve a pointer to a function, we must use a special relocation type.
   bool IsFunction = IsFunctionExpr(Fixup.getValue());
 
   switch (unsigned(Fixup.getKind())) {
-  case WebAssembly::fixup_code_global_index:
-    return wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB;
   case WebAssembly::fixup_code_sleb128_i32:
     if (IsFunction)
       return wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB;
@@ -98,10 +105,14 @@ WebAssemblyWasmObjectWriter::getRelocType(const MCValue &Target,
   case WebAssembly::fixup_code_sleb128_i64:
     llvm_unreachable("fixup_sleb128_i64 not implemented yet");
   case WebAssembly::fixup_code_uleb128_i32:
+    if (IsGlobalType(Target))
+      return wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB;
     if (IsFunctionType(Target))
       return wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB;
     if (IsFunction)
       return wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB;
+    if (IsEventType(Target))
+      return wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB;
     return wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB;
   case FK_Data_4:
     if (IsFunction)
diff --git a/lib/Target/WebAssembly/README.txt b/lib/Target/WebAssembly/README.txt
index ef0099f07efb..a154b4bf7ea8 100644
--- a/lib/Target/WebAssembly/README.txt
+++ b/lib/Target/WebAssembly/README.txt
@@ -94,10 +94,10 @@ WebAssemblyTargetLowering.
 //===---------------------------------------------------------------------===//
 
 Instead of the OptimizeReturned pass, which should consider preserving the
-"returned" attribute through to MachineInstrs and extending the StoreResults
-pass to do this optimization on calls too. That would also let the
-WebAssemblyPeephole pass clean up dead defs for such calls, as it does for
-stores.
+"returned" attribute through to MachineInstrs and extending the
+MemIntrinsicResults pass to do this optimization on calls too. That would also
+let the WebAssemblyPeephole pass clean up dead defs for such calls, as it does
+for stores.
 
 //===---------------------------------------------------------------------===//
 
@@ -120,8 +120,8 @@ code like this:
 It could be done with a smaller encoding like this:
 
     i32.const   $push5=, 0
-    tee_local   $push6=, $4=, $pop5
-    copy_local  $3=, $pop6
+    local.tee   $push6=, $4=, $pop5
+    local.copy  $3=, $pop6
 
 //===---------------------------------------------------------------------===//
 
@@ -180,11 +180,11 @@ floating-point constants.
 //===---------------------------------------------------------------------===//
 
 The function @dynamic_alloca_redzone in test/CodeGen/WebAssembly/userstack.ll
-ends up with a tee_local in its prolog which has an unused result, requiring
+ends up with a local.tee in its prolog which has an unused result, requiring
 an extra drop:
 
-    get_global  $push8=, 0
-    tee_local   $push9=, 1, $pop8
+    global.get  $push8=, 0
+    local.tee   $push9=, 1, $pop8
     drop        $pop9
     [...]
 
diff --git a/lib/Target/WebAssembly/WebAssembly.h b/lib/Target/WebAssembly/WebAssembly.h
index 05b7b21fb597..45145c0a6527 100644
--- a/lib/Target/WebAssembly/WebAssembly.h
+++ b/lib/Target/WebAssembly/WebAssembly.h
@@ -39,10 +39,11 @@ FunctionPass *createWebAssemblyArgumentMove();
 FunctionPass *createWebAssemblySetP2AlignOperands();
 
 // Late passes.
+FunctionPass *createWebAssemblyEHRestoreStackPointer();
 FunctionPass *createWebAssemblyReplacePhysRegs();
 FunctionPass *createWebAssemblyPrepareForLiveIntervals();
 FunctionPass *createWebAssemblyOptimizeLiveIntervals();
-FunctionPass *createWebAssemblyStoreResults();
+FunctionPass *createWebAssemblyMemIntrinsicResults();
 FunctionPass *createWebAssemblyRegStackify();
 FunctionPass *createWebAssemblyRegColoring();
 FunctionPass *createWebAssemblyExplicitLocals();
@@ -63,10 +64,11 @@ void initializeFixFunctionBitcastsPass(PassRegistry &);
 void initializeOptimizeReturnedPass(PassRegistry &);
 void initializeWebAssemblyArgumentMovePass(PassRegistry &);
 void initializeWebAssemblySetP2AlignOperandsPass(PassRegistry &);
+void initializeWebAssemblyEHRestoreStackPointerPass(PassRegistry &);
 void initializeWebAssemblyReplacePhysRegsPass(PassRegistry &);
 void initializeWebAssemblyPrepareForLiveIntervalsPass(PassRegistry &);
 void initializeWebAssemblyOptimizeLiveIntervalsPass(PassRegistry &);
-void initializeWebAssemblyStoreResultsPass(PassRegistry &);
+void initializeWebAssemblyMemIntrinsicResultsPass(PassRegistry &);
 void initializeWebAssemblyRegStackifyPass(PassRegistry &);
 void initializeWebAssemblyRegColoringPass(PassRegistry &);
 void initializeWebAssemblyExplicitLocalsPass(PassRegistry &);
diff --git a/lib/Target/WebAssembly/WebAssembly.td b/lib/Target/WebAssembly/WebAssembly.td
index 2f301da8e422..6b218f8aa880 100644
--- a/lib/Target/WebAssembly/WebAssembly.td
+++ b/lib/Target/WebAssembly/WebAssembly.td
@@ -23,8 +23,15 @@ include "llvm/Target/Target.td"
 // WebAssembly Subtarget features.
 //===----------------------------------------------------------------------===//
 
-def FeatureSIMD128 : SubtargetFeature<"simd128", "HasSIMD128", "true",
+def FeatureSIMD128 : SubtargetFeature<"simd128", "SIMDLevel", "SIMD128",
                                       "Enable 128-bit SIMD">;
+
+def FeatureUnimplementedSIMD128 :
+      SubtargetFeature<"unimplemented-simd128",
+                       "SIMDLevel", "UnimplementedSIMD128",
+                       "Enable 128-bit SIMD not yet implemented in engines",
+                       [FeatureSIMD128]>;
+
 def FeatureAtomics : SubtargetFeature<"atomics", "HasAtomics", "true",
                                       "Enable Atomics">;
 def FeatureNontrappingFPToInt :
@@ -71,7 +78,8 @@ def : ProcessorModel<"generic", NoSchedModel, []>;
 
 // Latest and greatest experimental version of WebAssembly. Bugs included!
 def : ProcessorModel<"bleeding-edge", NoSchedModel,
-                      [FeatureSIMD128, FeatureAtomics]>;
+                      [FeatureSIMD128, FeatureAtomics,
+                       FeatureNontrappingFPToInt, FeatureSignExt]>;
 
 //===----------------------------------------------------------------------===//
 // Target Declaration
diff --git a/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp b/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
index 4af9cd150bf7..e49e2b67f435 100644
--- a/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
@@ -24,10 +24,10 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "wasm-add-missing-prototypes"
@@ -60,16 +60,17 @@ ModulePass *llvm::createWebAssemblyAddMissingPrototypes() {
 }
 
 bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
-  LLVM_DEBUG(dbgs() << "runnning AddMissingPrototypes\n");
+  LLVM_DEBUG(dbgs() << "********** Add Missing Prototypes **********\n");
 
-  std::vector<std::pair<Function*, Function*>> Replacements;
+  std::vector<std::pair<Function *, Function *>> Replacements;
 
   // Find all the prototype-less function declarations
   for (Function &F : M) {
     if (!F.isDeclaration() || !F.hasFnAttribute("no-prototype"))
       continue;
 
-    LLVM_DEBUG(dbgs() << "Found no-prototype function: " << F.getName() << "\n");
+    LLVM_DEBUG(dbgs() << "Found no-prototype function: " << F.getName()
+                      << "\n");
 
     // When clang emits prototype-less C functions it uses (...), i.e. varargs
     // function that take no arguments (have no sentinel).  When we see a
@@ -83,23 +84,29 @@ bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
           "Functions with 'no-prototype' attribute should not have params: " +
           F.getName());
 
-
     // Create a function prototype based on the first call site (first bitcast)
     // that we find.
     FunctionType *NewType = nullptr;
-    Function* NewF = nullptr;
+    Function *NewF = nullptr;
     for (Use &U : F.uses()) {
       LLVM_DEBUG(dbgs() << "prototype-less use: " << F.getName() << "\n");
-      if (BitCastOperator *BC = dyn_cast<BitCastOperator>(U.getUser())) {
-        FunctionType *DestType =
-            cast<FunctionType>(BC->getDestTy()->getPointerElementType());
-
-        // Create a new function with the correct type
-        NewType = DestType;
-        NewF = Function::Create(NewType, F.getLinkage(), F.getName());
-        NewF->setAttributes(F.getAttributes());
-        NewF->removeFnAttr("no-prototype");
-        break;
+      if (auto *BC = dyn_cast<BitCastOperator>(U.getUser())) {
+        if (auto *DestType = dyn_cast<FunctionType>(
+                BC->getDestTy()->getPointerElementType())) {
+          if (!NewType) {
+            // Create a new function with the correct type
+            NewType = DestType;
+            NewF = Function::Create(NewType, F.getLinkage(), F.getName());
+            NewF->setAttributes(F.getAttributes());
+            NewF->removeFnAttr("no-prototype");
+          } else {
+            if (NewType != DestType) {
+              report_fatal_error("Prototypeless function used with "
+                                 "conflicting signatures: " +
+                                 F.getName());
+            }
+          }
+        }
       }
     }
 
@@ -110,32 +117,42 @@ bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
       continue;
     }
 
-    for (Use &U : F.uses()) {
-      if (BitCastOperator *BC = dyn_cast<BitCastOperator>(U.getUser())) {
-        FunctionType *DestType =
-            cast<FunctionType>(BC->getDestTy()->getPointerElementType());
-        if (NewType != DestType) {
-          report_fatal_error(
-              "Prototypeless function used with conflicting signatures: " +
-              F.getName());
-        }
-        BC->replaceAllUsesWith(NewF);
-        Replacements.emplace_back(&F, NewF);
-      } else {
-        dbgs() << *U.getUser()->getType() << "\n";
+    SmallVector<Instruction *, 4> DeadInsts;
+
+    for (Use &US : F.uses()) {
+      User *U = US.getUser();
+      if (auto *BC = dyn_cast<BitCastOperator>(U)) {
+        if (auto *Inst = dyn_cast<BitCastInst>(U)) {
+          // Replace with a new bitcast
+          IRBuilder<> Builder(Inst);
+          Value *NewCast = Builder.CreatePointerCast(NewF, BC->getDestTy());
+          Inst->replaceAllUsesWith(NewCast);
+          DeadInsts.push_back(Inst);
+        } else if (auto *Const = dyn_cast<ConstantExpr>(U)) {
+          Constant *NewConst =
+              ConstantExpr::getPointerCast(NewF, BC->getDestTy());
+          Const->replaceAllUsesWith(NewConst);
+        } else {
+          dbgs() << *U->getType() << "\n";
 #ifndef NDEBUG
-        U.getUser()->dump();
+          U->dump();
 #endif
-        report_fatal_error(
-            "unexpected use of prototypeless function: " + F.getName() + "\n");
+          report_fatal_error("unexpected use of prototypeless function: " +
+                             F.getName() + "\n");
+        }
       }
     }
+
+    for (auto I : DeadInsts)
+      I->eraseFromParent();
+    Replacements.emplace_back(&F, NewF);
   }
 
+
   // Finally replace the old function declarations with the new ones
   for (auto &Pair : Replacements) {
-    Function* Old = Pair.first;
-    Function* New = Pair.second;
+    Function *Old = Pair.first;
+    Function *New = Pair.second;
     Old->eraseFromParent();
     M.getFunctionList().push_back(New);
   }
diff --git a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
index 1f280e1d13fc..c4f03dfa7f9e 100644
--- a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
@@ -50,7 +50,7 @@ MVT WebAssemblyAsmPrinter::getRegType(unsigned RegNo) const {
   const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
   const TargetRegisterClass *TRC = MRI->getRegClass(RegNo);
   for (MVT T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64, MVT::v16i8, MVT::v8i16,
-                MVT::v4i32, MVT::v4f32})
+                MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64})
     if (TRI->isTypeLegalForClass(*TRC, T))
       return T;
   LLVM_DEBUG(errs() << "Unknown type for register number: " << RegNo);
@@ -78,24 +78,45 @@ WebAssemblyTargetStreamer *WebAssemblyAsmPrinter::getTargetStreamer() {
 //===----------------------------------------------------------------------===//
 
 void WebAssemblyAsmPrinter::EmitEndOfAsmFile(Module &M) {
+  for (auto &It : OutContext.getSymbols()) {
+    // Emit a .globaltype and .eventtype declaration.
+    auto Sym = cast<MCSymbolWasm>(It.getValue());
+    if (Sym->getType() == wasm::WASM_SYMBOL_TYPE_GLOBAL)
+      getTargetStreamer()->emitGlobalType(Sym);
+    else if (Sym->getType() == wasm::WASM_SYMBOL_TYPE_EVENT)
+      getTargetStreamer()->emitEventType(Sym);
+  }
+
   for (const auto &F : M) {
     // Emit function type info for all undefined functions
     if (F.isDeclarationForLinker() && !F.isIntrinsic()) {
       SmallVector<MVT, 4> Results;
       SmallVector<MVT, 4> Params;
-      ComputeSignatureVTs(F, TM, Params, Results);
-      MCSymbol *Sym = getSymbol(&F);
-      getTargetStreamer()->emitIndirectFunctionType(Sym, Params, Results);
+      ComputeSignatureVTs(F.getFunctionType(), F, TM, Params, Results);
+      auto *Sym = cast<MCSymbolWasm>(getSymbol(&F));
+      Sym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
+      if (!Sym->getSignature()) {
+        auto Signature = SignatureFromMVTs(Results, Params);
+        Sym->setSignature(Signature.get());
+        addSignature(std::move(Signature));
+      }
+      // FIXME: this was originally intended for post-linking and was only used
+      // for imports that were only called indirectly (i.e. s2wasm could not
+      // infer the type from a call). With object files it applies to all
+      // imports. so fix the names and the tests, or rethink how import
+      // delcarations work in asm files.
+      getTargetStreamer()->emitFunctionType(Sym);
 
       if (TM.getTargetTriple().isOSBinFormatWasm() &&
           F.hasFnAttribute("wasm-import-module")) {
-        MCSymbolWasm *WasmSym = cast<MCSymbolWasm>(Sym);
-        StringRef Name = F.getFnAttribute("wasm-import-module")
-                             .getValueAsString();
-        getTargetStreamer()->emitImportModule(WasmSym, Name);
+        StringRef Name =
+            F.getFnAttribute("wasm-import-module").getValueAsString();
+        Sym->setModuleName(Name);
+        getTargetStreamer()->emitImportModule(Sym, Name);
       }
     }
   }
+
   for (const auto &G : M.globals()) {
     if (!G.hasInitializer() && G.hasExternalLinkage()) {
       if (G.getValueType()->isSized()) {
@@ -137,10 +158,18 @@ void WebAssemblyAsmPrinter::EmitJumpTableInfo() {
 }
 
 void WebAssemblyAsmPrinter::EmitFunctionBodyStart() {
-  getTargetStreamer()->emitParam(CurrentFnSym, MFI->getParams());
-
-  SmallVector<MVT, 4> ResultVTs;
   const Function &F = MF->getFunction();
+  SmallVector<MVT, 1> ResultVTs;
+  SmallVector<MVT, 4> ParamVTs;
+  ComputeSignatureVTs(F.getFunctionType(), F, TM, ParamVTs, ResultVTs);
+  auto Signature = SignatureFromMVTs(ResultVTs, ParamVTs);
+  auto *WasmSym = cast<MCSymbolWasm>(CurrentFnSym);
+  WasmSym->setSignature(Signature.get());
+  addSignature(std::move(Signature));
+  WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
+
+  // FIXME: clean up how params and results are emitted (use signatures)
+  getTargetStreamer()->emitFunctionType(WasmSym);
 
   // Emit the function index.
   if (MDNode *Idx = F.getMetadata("wasm.index")) {
@@ -150,16 +179,9 @@ void WebAssemblyAsmPrinter::EmitFunctionBodyStart() {
         cast<ConstantAsMetadata>(Idx->getOperand(0))->getValue()));
   }
 
-  ComputeLegalValueVTs(F, TM, F.getReturnType(), ResultVTs);
-
-  // If the return type needs to be legalized it will get converted into
-  // passing a pointer.
-  if (ResultVTs.size() == 1)
-    getTargetStreamer()->emitResult(CurrentFnSym, ResultVTs);
-  else
-    getTargetStreamer()->emitResult(CurrentFnSym, ArrayRef<MVT>());
-
-  getTargetStreamer()->emitLocal(MFI->getLocals());
+  SmallVector<wasm::ValType, 16> Locals;
+  ValTypesFromMVTs(MFI->getLocals(), Locals);
+  getTargetStreamer()->emitLocal(Locals);
 
   AsmPrinter::EmitFunctionBodyStart();
 }
@@ -168,42 +190,63 @@ void WebAssemblyAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   LLVM_DEBUG(dbgs() << "EmitInstruction: " << *MI << '\n');
 
   switch (MI->getOpcode()) {
-  case WebAssembly::ARGUMENT_I32:
-  case WebAssembly::ARGUMENT_I64:
-  case WebAssembly::ARGUMENT_F32:
-  case WebAssembly::ARGUMENT_F64:
+  case WebAssembly::ARGUMENT_i32:
+  case WebAssembly::ARGUMENT_i32_S:
+  case WebAssembly::ARGUMENT_i64:
+  case WebAssembly::ARGUMENT_i64_S:
+  case WebAssembly::ARGUMENT_f32:
+  case WebAssembly::ARGUMENT_f32_S:
+  case WebAssembly::ARGUMENT_f64:
+  case WebAssembly::ARGUMENT_f64_S:
   case WebAssembly::ARGUMENT_v16i8:
+  case WebAssembly::ARGUMENT_v16i8_S:
   case WebAssembly::ARGUMENT_v8i16:
+  case WebAssembly::ARGUMENT_v8i16_S:
   case WebAssembly::ARGUMENT_v4i32:
+  case WebAssembly::ARGUMENT_v4i32_S:
+  case WebAssembly::ARGUMENT_v2i64:
+  case WebAssembly::ARGUMENT_v2i64_S:
   case WebAssembly::ARGUMENT_v4f32:
+  case WebAssembly::ARGUMENT_v4f32_S:
+  case WebAssembly::ARGUMENT_v2f64:
+  case WebAssembly::ARGUMENT_v2f64_S:
     // These represent values which are live into the function entry, so there's
     // no instruction to emit.
     break;
   case WebAssembly::FALLTHROUGH_RETURN_I32:
+  case WebAssembly::FALLTHROUGH_RETURN_I32_S:
   case WebAssembly::FALLTHROUGH_RETURN_I64:
+  case WebAssembly::FALLTHROUGH_RETURN_I64_S:
   case WebAssembly::FALLTHROUGH_RETURN_F32:
+  case WebAssembly::FALLTHROUGH_RETURN_F32_S:
   case WebAssembly::FALLTHROUGH_RETURN_F64:
+  case WebAssembly::FALLTHROUGH_RETURN_F64_S:
   case WebAssembly::FALLTHROUGH_RETURN_v16i8:
+  case WebAssembly::FALLTHROUGH_RETURN_v16i8_S:
   case WebAssembly::FALLTHROUGH_RETURN_v8i16:
+  case WebAssembly::FALLTHROUGH_RETURN_v8i16_S:
   case WebAssembly::FALLTHROUGH_RETURN_v4i32:
-  case WebAssembly::FALLTHROUGH_RETURN_v4f32: {
+  case WebAssembly::FALLTHROUGH_RETURN_v4i32_S:
+  case WebAssembly::FALLTHROUGH_RETURN_v2i64:
+  case WebAssembly::FALLTHROUGH_RETURN_v2i64_S:
+  case WebAssembly::FALLTHROUGH_RETURN_v4f32:
+  case WebAssembly::FALLTHROUGH_RETURN_v4f32_S:
+  case WebAssembly::FALLTHROUGH_RETURN_v2f64:
+  case WebAssembly::FALLTHROUGH_RETURN_v2f64_S: {
     // These instructions represent the implicit return at the end of a
-    // function body. The operand is always a pop.
-    assert(MFI->isVRegStackified(MI->getOperand(0).getReg()));
-
+    // function body. Always pops one value off the stack.
     if (isVerbose()) {
-      OutStreamer->AddComment("fallthrough-return: $pop" +
-                              Twine(MFI->getWARegStackId(
-                                  MFI->getWAReg(MI->getOperand(0).getReg()))));
+      OutStreamer->AddComment("fallthrough-return-value");
       OutStreamer->AddBlankLine();
     }
     break;
   }
   case WebAssembly::FALLTHROUGH_RETURN_VOID:
+  case WebAssembly::FALLTHROUGH_RETURN_VOID_S:
     // This instruction represents the implicit return at the end of a
     // function body with no return value.
     if (isVerbose()) {
-      OutStreamer->AddComment("fallthrough-return");
+      OutStreamer->AddComment("fallthrough-return-void");
       OutStreamer->AddBlankLine();
     }
     break;
@@ -244,6 +287,9 @@ bool WebAssemblyAsmPrinter::PrintAsmOperand(const MachineInstr *MI,
       OS << MO.getImm();
       return false;
     case MachineOperand::MO_Register:
+      // FIXME: only opcode that still contains registers, as required by
+      // MachineInstr::getDebugVariable().
+      assert(MI->getOpcode() == WebAssembly::INLINEASM);
       OS << regToString(MO);
       return false;
     case MachineOperand::MO_GlobalAddress:
diff --git a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
index 23817b4e5126..f6cb5610bad3 100644
--- a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
+++ b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
@@ -25,18 +25,23 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter {
   const WebAssemblySubtarget *Subtarget;
   const MachineRegisterInfo *MRI;
   WebAssemblyFunctionInfo *MFI;
+  // TODO: Do the uniquing of Signatures here instead of ObjectFileWriter?
+  std::vector<std::unique_ptr<wasm::WasmSignature>> Signatures;
 
 public:
   explicit WebAssemblyAsmPrinter(TargetMachine &TM,
                                  std::unique_ptr<MCStreamer> Streamer)
-      : AsmPrinter(TM, std::move(Streamer)),
-        Subtarget(nullptr), MRI(nullptr), MFI(nullptr) {}
+      : AsmPrinter(TM, std::move(Streamer)), Subtarget(nullptr), MRI(nullptr),
+        MFI(nullptr) {}
 
   StringRef getPassName() const override {
     return "WebAssembly Assembly Printer";
   }
 
   const WebAssemblySubtarget &getSubtarget() const { return *Subtarget; }
+  void addSignature(std::unique_ptr<wasm::WasmSignature> &&Sig) {
+    Signatures.push_back(std::move(Sig));
+  }
 
   //===------------------------------------------------------------------===//
   // MachineFunctionPass Implementation.
diff --git a/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp b/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
index 267a51433cd1..fc827e9d5780 100644
--- a/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
@@ -11,14 +11,15 @@
 /// This file implements a CFG sorting pass.
 ///
 /// This pass reorders the blocks in a function to put them into topological
-/// order, ignoring loop backedges, and without any loop being interrupted
-/// by a block not dominated by the loop header, with special care to keep the
-/// order as similar as possible to the original order.
+/// order, ignoring loop backedges, and without any loop or exception being
+/// interrupted by a block not dominated by the its header, with special care
+/// to keep the order as similar as possible to the original order.
 ///
 ////===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "WebAssembly.h"
+#include "WebAssemblyExceptionInfo.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/PriorityQueue.h"
@@ -35,6 +36,73 @@ using namespace llvm;
 #define DEBUG_TYPE "wasm-cfg-sort"
 
 namespace {
+
+// Wrapper for loops and exceptions
+class Region {
+public:
+  virtual ~Region() = default;
+  virtual MachineBasicBlock *getHeader() const = 0;
+  virtual bool contains(const MachineBasicBlock *MBB) const = 0;
+  virtual unsigned getNumBlocks() const = 0;
+  using block_iterator = typename ArrayRef<MachineBasicBlock *>::const_iterator;
+  virtual iterator_range<block_iterator> blocks() const = 0;
+  virtual bool isLoop() const = 0;
+};
+
+template <typename T> class ConcreteRegion : public Region {
+  const T *Region;
+
+public:
+  ConcreteRegion(const T *Region) : Region(Region) {}
+  MachineBasicBlock *getHeader() const override { return Region->getHeader(); }
+  bool contains(const MachineBasicBlock *MBB) const override {
+    return Region->contains(MBB);
+  }
+  unsigned getNumBlocks() const override { return Region->getNumBlocks(); }
+  iterator_range<block_iterator> blocks() const override {
+    return Region->blocks();
+  }
+  bool isLoop() const override { return false; }
+};
+
+template <> bool ConcreteRegion<MachineLoop>::isLoop() const { return true; }
+
+// This class has information of nested Regions; this is analogous to what
+// LoopInfo is for loops.
+class RegionInfo {
+  const MachineLoopInfo &MLI;
+  const WebAssemblyExceptionInfo &WEI;
+  std::vector<const Region *> Regions;
+  DenseMap<const MachineLoop *, std::unique_ptr<Region>> LoopMap;
+  DenseMap<const WebAssemblyException *, std::unique_ptr<Region>> ExceptionMap;
+
+public:
+  RegionInfo(const MachineLoopInfo &MLI, const WebAssemblyExceptionInfo &WEI)
+      : MLI(MLI), WEI(WEI) {}
+
+  // Returns a smallest loop or exception that contains MBB
+  const Region *getRegionFor(const MachineBasicBlock *MBB) {
+    const auto *ML = MLI.getLoopFor(MBB);
+    const auto *WE = WEI.getExceptionFor(MBB);
+    if (!ML && !WE)
+      return nullptr;
+    if ((ML && !WE) || (ML && WE && ML->getNumBlocks() < WE->getNumBlocks())) {
+      // If the smallest region containing MBB is a loop
+      if (LoopMap.count(ML))
+        return LoopMap[ML].get();
+      LoopMap[ML] = llvm::make_unique<ConcreteRegion<MachineLoop>>(ML);
+      return LoopMap[ML].get();
+    } else {
+      // If the smallest region containing MBB is an exception
+      if (ExceptionMap.count(WE))
+        return ExceptionMap[WE].get();
+      ExceptionMap[WE] =
+          llvm::make_unique<ConcreteRegion<WebAssemblyException>>(WE);
+      return ExceptionMap[WE].get();
+    }
+  }
+};
+
 class WebAssemblyCFGSort final : public MachineFunctionPass {
   StringRef getPassName() const override { return "WebAssembly CFG Sort"; }
 
@@ -44,6 +112,8 @@ class WebAssemblyCFGSort final : public MachineFunctionPass {
     AU.addPreserved<MachineDominatorTree>();
     AU.addRequired<MachineLoopInfo>();
     AU.addPreserved<MachineLoopInfo>();
+    AU.addRequired<WebAssemblyExceptionInfo>();
+    AU.addPreserved<WebAssemblyExceptionInfo>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
@@ -81,10 +151,48 @@ static void MaybeUpdateTerminator(MachineBasicBlock *MBB) {
 }
 
 namespace {
+// EH pads are selected first regardless of the block comparison order.
+// When only one of the BBs is an EH pad, we give a higher priority to it, to
+// prevent common mismatches between possibly throwing calls and ehpads they
+// unwind to, as in the example below:
+//
+// bb0:
+//   call @foo      // If this throws, unwind to bb2
+// bb1:
+//   call @bar      // If this throws, unwind to bb3
+// bb2 (ehpad):
+//   handler_bb2
+// bb3 (ehpad):
+//   handler_bb3
+// continuing code
+//
+// Because this pass tries to preserve the original BB order, this order will
+// not change. But this will result in this try-catch structure in CFGStackify,
+// resulting in a mismatch:
+// try
+//   try
+//     call @foo
+//     call @bar    // This should unwind to bb3, not bb2!
+//   catch
+//     handler_bb2
+//   end
+// catch
+//   handler_bb3
+// end
+// continuing code
+//
+// If we give a higher priority to an EH pad whenever it is ready in this
+// example, when both bb1 and bb2 are ready, we would pick up bb2 first.
+
 /// Sort blocks by their number.
 struct CompareBlockNumbers {
   bool operator()(const MachineBasicBlock *A,
                   const MachineBasicBlock *B) const {
+    if (A->isEHPad() && !B->isEHPad())
+      return false;
+    if (!A->isEHPad() && B->isEHPad())
+      return true;
+
     return A->getNumber() > B->getNumber();
   }
 };
@@ -92,29 +200,36 @@ struct CompareBlockNumbers {
 struct CompareBlockNumbersBackwards {
   bool operator()(const MachineBasicBlock *A,
                   const MachineBasicBlock *B) const {
+    // We give a higher priority to an EH pad
+    if (A->isEHPad() && !B->isEHPad())
+      return false;
+    if (!A->isEHPad() && B->isEHPad())
+      return true;
+
     return A->getNumber() < B->getNumber();
   }
 };
-/// Bookkeeping for a loop to help ensure that we don't mix blocks not dominated
-/// by the loop header among the loop's blocks.
+/// Bookkeeping for a region to help ensure that we don't mix blocks not
+/// dominated by the its header among its blocks.
 struct Entry {
-  const MachineLoop *Loop;
+  const Region *TheRegion;
   unsigned NumBlocksLeft;
 
   /// List of blocks not dominated by Loop's header that are deferred until
   /// after all of Loop's blocks have been seen.
   std::vector<MachineBasicBlock *> Deferred;
 
-  explicit Entry(const MachineLoop *L)
-      : Loop(L), NumBlocksLeft(L->getNumBlocks()) {}
+  explicit Entry(const class Region *R)
+      : TheRegion(R), NumBlocksLeft(R->getNumBlocks()) {}
 };
 } // end anonymous namespace
 
-/// Sort the blocks, taking special care to make sure that loops are not
+/// Sort the blocks, taking special care to make sure that regions are not
 /// interrupted by blocks not dominated by their header.
 /// TODO: There are many opportunities for improving the heuristics here.
 /// Explore them.
 static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
+                       const WebAssemblyExceptionInfo &WEI,
                        const MachineDominatorTree &MDT) {
   // Prepare for a topological sort: Record the number of predecessors each
   // block has, ignoring loop backedges.
@@ -131,35 +246,39 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
   }
 
   // Topological sort the CFG, with additional constraints:
-  //  - Between a loop header and the last block in the loop, there can be
-  //    no blocks not dominated by the loop header.
+  //  - Between a region header and the last block in the region, there can be
+  //    no blocks not dominated by its header.
   //  - It's desirable to preserve the original block order when possible.
   // We use two ready lists; Preferred and Ready. Preferred has recently
   // processed successors, to help preserve block sequences from the original
-  // order. Ready has the remaining ready blocks.
+  // order. Ready has the remaining ready blocks. EH blocks are picked first
+  // from both queues.
   PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>,
                 CompareBlockNumbers>
       Preferred;
   PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>,
                 CompareBlockNumbersBackwards>
       Ready;
-  SmallVector<Entry, 4> Loops;
+
+  RegionInfo SUI(MLI, WEI);
+  SmallVector<Entry, 4> Entries;
   for (MachineBasicBlock *MBB = &MF.front();;) {
-    const MachineLoop *L = MLI.getLoopFor(MBB);
-    if (L) {
-      // If MBB is a loop header, add it to the active loop list. We can't put
-      // any blocks that it doesn't dominate until we see the end of the loop.
-      if (L->getHeader() == MBB)
-        Loops.push_back(Entry(L));
-      // For each active loop the block is in, decrement the count. If MBB is
-      // the last block in an active loop, take it off the list and pick up any
-      // blocks deferred because the header didn't dominate them.
-      for (Entry &E : Loops)
-        if (E.Loop->contains(MBB) && --E.NumBlocksLeft == 0)
+    const Region *R = SUI.getRegionFor(MBB);
+    if (R) {
+      // If MBB is a region header, add it to the active region list. We can't
+      // put any blocks that it doesn't dominate until we see the end of the
+      // region.
+      if (R->getHeader() == MBB)
+        Entries.push_back(Entry(R));
+      // For each active region the block is in, decrement the count. If MBB is
+      // the last block in an active region, take it off the list and pick up
+      // any blocks deferred because the header didn't dominate them.
+      for (Entry &E : Entries)
+        if (E.TheRegion->contains(MBB) && --E.NumBlocksLeft == 0)
           for (auto DeferredBlock : E.Deferred)
             Ready.push(DeferredBlock);
-      while (!Loops.empty() && Loops.back().NumBlocksLeft == 0)
-        Loops.pop_back();
+      while (!Entries.empty() && Entries.back().NumBlocksLeft == 0)
+        Entries.pop_back();
     }
     // The main topological sort logic.
     for (MachineBasicBlock *Succ : MBB->successors()) {
@@ -177,19 +296,19 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
     while (!Preferred.empty()) {
       Next = Preferred.top();
       Preferred.pop();
-      // If X isn't dominated by the top active loop header, defer it until that
-      // loop is done.
-      if (!Loops.empty() &&
-          !MDT.dominates(Loops.back().Loop->getHeader(), Next)) {
-        Loops.back().Deferred.push_back(Next);
+      // If X isn't dominated by the top active region header, defer it until
+      // that region is done.
+      if (!Entries.empty() &&
+          !MDT.dominates(Entries.back().TheRegion->getHeader(), Next)) {
+        Entries.back().Deferred.push_back(Next);
         Next = nullptr;
         continue;
       }
       // If Next was originally ordered before MBB, and it isn't because it was
       // loop-rotated above the header, it's not preferred.
       if (Next->getNumber() < MBB->getNumber() &&
-          (!L || !L->contains(Next) ||
-           L->getHeader()->getNumber() < Next->getNumber())) {
+          (!R || !R->contains(Next) ||
+           R->getHeader()->getNumber() < Next->getNumber())) {
         Ready.push(Next);
         Next = nullptr;
         continue;
@@ -207,11 +326,11 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
       for (;;) {
         Next = Ready.top();
         Ready.pop();
-        // If Next isn't dominated by the top active loop header, defer it until
-        // that loop is done.
-        if (!Loops.empty() &&
-            !MDT.dominates(Loops.back().Loop->getHeader(), Next)) {
-          Loops.back().Deferred.push_back(Next);
+        // If Next isn't dominated by the top active region header, defer it
+        // until that region is done.
+        if (!Entries.empty() &&
+            !MDT.dominates(Entries.back().TheRegion->getHeader(), Next)) {
+          Entries.back().Deferred.push_back(Next);
           continue;
         }
         break;
@@ -222,11 +341,11 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
     MaybeUpdateTerminator(MBB);
     MBB = Next;
   }
-  assert(Loops.empty() && "Active loop list not finished");
+  assert(Entries.empty() && "Active sort region list not finished");
   MF.RenumberBlocks();
 
 #ifndef NDEBUG
-  SmallSetVector<MachineLoop *, 8> OnStack;
+  SmallSetVector<const Region *, 8> OnStack;
 
   // Insert a sentinel representing the degenerate loop that starts at the
   // function entry block and includes the entire function as a "loop" that
@@ -235,29 +354,39 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
 
   for (auto &MBB : MF) {
     assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative.");
+    const Region *Region = SUI.getRegionFor(&MBB);
+
+    if (Region && &MBB == Region->getHeader()) {
+      if (Region->isLoop()) {
+        // Loop header. The loop predecessor should be sorted above, and the
+        // other predecessors should be backedges below.
+        for (auto Pred : MBB.predecessors())
+          assert(
+              (Pred->getNumber() < MBB.getNumber() || Region->contains(Pred)) &&
+              "Loop header predecessors must be loop predecessors or "
+              "backedges");
+      } else {
+        // Not a loop header. All predecessors should be sorted above.
+        for (auto Pred : MBB.predecessors())
+          assert(Pred->getNumber() < MBB.getNumber() &&
+                 "Non-loop-header predecessors should be topologically sorted");
+      }
+      assert(OnStack.insert(Region) &&
+             "Regions should be declared at most once.");
 
-    MachineLoop *Loop = MLI.getLoopFor(&MBB);
-    if (Loop && &MBB == Loop->getHeader()) {
-      // Loop header. The loop predecessor should be sorted above, and the other
-      // predecessors should be backedges below.
-      for (auto Pred : MBB.predecessors())
-        assert(
-            (Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) &&
-            "Loop header predecessors must be loop predecessors or backedges");
-      assert(OnStack.insert(Loop) && "Loops should be declared at most once.");
     } else {
       // Not a loop header. All predecessors should be sorted above.
       for (auto Pred : MBB.predecessors())
         assert(Pred->getNumber() < MBB.getNumber() &&
                "Non-loop-header predecessors should be topologically sorted");
-      assert(OnStack.count(MLI.getLoopFor(&MBB)) &&
-             "Blocks must be nested in their loops");
+      assert(OnStack.count(SUI.getRegionFor(&MBB)) &&
+             "Blocks must be nested in their regions");
     }
     while (OnStack.size() > 1 && &MBB == WebAssembly::getBottom(OnStack.back()))
       OnStack.pop_back();
   }
   assert(OnStack.pop_back_val() == nullptr &&
-         "The function entry block shouldn't actually be a loop header");
+         "The function entry block shouldn't actually be a region header");
   assert(OnStack.empty() &&
          "Control flow stack pushes and pops should be balanced.");
 #endif
@@ -269,12 +398,13 @@ bool WebAssemblyCFGSort::runOnMachineFunction(MachineFunction &MF) {
                     << MF.getName() << '\n');
 
   const auto &MLI = getAnalysis<MachineLoopInfo>();
+  const auto &WEI = getAnalysis<WebAssemblyExceptionInfo>();
   auto &MDT = getAnalysis<MachineDominatorTree>();
   // Liveness is not tracked for VALUE_STACK physreg.
   MF.getRegInfo().invalidateLiveness();
 
-  // Sort the blocks, with contiguous loops.
-  SortBlocks(MF, MLI, MDT);
+  // Sort the blocks, with contiguous sort regions.
+  SortBlocks(MF, MLI, WEI, MDT);
 
   return true;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 70ce40cefed7..f8f5f4040c86 100644
--- a/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -10,16 +10,21 @@
 /// \file
 /// This file implements a CFG stacking pass.
 ///
-/// This pass inserts BLOCK and LOOP markers to mark the start of scopes, since
-/// scope boundaries serve as the labels for WebAssembly's control transfers.
+/// This pass inserts BLOCK, LOOP, and TRY markers to mark the start of scopes,
+/// since scope boundaries serve as the labels for WebAssembly's control
+/// transfers.
 ///
 /// This is sufficient to convert arbitrary CFGs into a form that works on
 /// WebAssembly, provided that all loops are single-entry.
 ///
+/// In case we use exceptions, this pass also fixes mismatches in unwind
+/// destinations created during transforming CFG into wasm structured format.
+///
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "WebAssembly.h"
+#include "WebAssemblyExceptionInfo.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
@@ -29,6 +34,8 @@
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/WasmEHFuncInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
@@ -40,26 +47,57 @@ class WebAssemblyCFGStackify final : public MachineFunctionPass {
   StringRef getPassName() const override { return "WebAssembly CFG Stackify"; }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
     AU.addRequired<MachineDominatorTree>();
-    AU.addPreserved<MachineDominatorTree>();
     AU.addRequired<MachineLoopInfo>();
-    AU.addPreserved<MachineLoopInfo>();
+    AU.addRequired<WebAssemblyExceptionInfo>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
+  // For each block whose label represents the end of a scope, record the block
+  // which holds the beginning of the scope. This will allow us to quickly skip
+  // over scoped regions when walking blocks.
+  SmallVector<MachineBasicBlock *, 8> ScopeTops;
+
+  void placeMarkers(MachineFunction &MF);
+  void placeBlockMarker(MachineBasicBlock &MBB);
+  void placeLoopMarker(MachineBasicBlock &MBB);
+  void placeTryMarker(MachineBasicBlock &MBB);
+  void rewriteDepthImmediates(MachineFunction &MF);
+  void fixEndsAtEndOfFunction(MachineFunction &MF);
+
+  // For each BLOCK|LOOP|TRY, the corresponding END_(BLOCK|LOOP|TRY).
+  DenseMap<const MachineInstr *, MachineInstr *> BeginToEnd;
+  // For each END_(BLOCK|LOOP|TRY), the corresponding BLOCK|LOOP|TRY.
+  DenseMap<const MachineInstr *, MachineInstr *> EndToBegin;
+  // <TRY marker, EH pad> map
+  DenseMap<const MachineInstr *, MachineBasicBlock *> TryToEHPad;
+  // <EH pad, TRY marker> map
+  DenseMap<const MachineBasicBlock *, MachineInstr *> EHPadToTry;
+  // <LOOP|TRY marker, Loop/exception bottom BB> map
+  DenseMap<const MachineInstr *, MachineBasicBlock *> BeginToBottom;
+
+  // Helper functions to register scope information created by marker
+  // instructions.
+  void registerScope(MachineInstr *Begin, MachineInstr *End);
+  void registerTryScope(MachineInstr *Begin, MachineInstr *End,
+                        MachineBasicBlock *EHPad);
+
+  MachineBasicBlock *getBottom(const MachineInstr *Begin);
+
 public:
   static char ID; // Pass identification, replacement for typeid
   WebAssemblyCFGStackify() : MachineFunctionPass(ID) {}
+  ~WebAssemblyCFGStackify() override { releaseMemory(); }
+  void releaseMemory() override;
 };
 } // end anonymous namespace
 
 char WebAssemblyCFGStackify::ID = 0;
 INITIALIZE_PASS(WebAssemblyCFGStackify, DEBUG_TYPE,
-                "Insert BLOCK and LOOP markers for WebAssembly scopes",
-                false, false)
+                "Insert BLOCK and LOOP markers for WebAssembly scopes", false,
+                false)
 
 FunctionPass *llvm::createWebAssemblyCFGStackify() {
   return new WebAssemblyCFGStackify();
@@ -73,34 +111,121 @@ FunctionPass *llvm::createWebAssemblyCFGStackify() {
 static bool ExplicitlyBranchesTo(MachineBasicBlock *Pred,
                                  MachineBasicBlock *MBB) {
   for (MachineInstr &MI : Pred->terminators())
-    for (MachineOperand &MO : MI.explicit_operands())
-      if (MO.isMBB() && MO.getMBB() == MBB)
-        return true;
+    // Even if a rethrow takes a BB argument, it is not a branch
+    if (!WebAssembly::isRethrow(MI))
+      for (MachineOperand &MO : MI.explicit_operands())
+        if (MO.isMBB() && MO.getMBB() == MBB)
+          return true;
   return false;
 }
 
+// Returns an iterator to the earliest position possible within the MBB,
+// satisfying the restrictions given by BeforeSet and AfterSet. BeforeSet
+// contains instructions that should go before the marker, and AfterSet contains
+// ones that should go after the marker. In this function, AfterSet is only
+// used for sanity checking.
+static MachineBasicBlock::iterator
+GetEarliestInsertPos(MachineBasicBlock *MBB,
+                     const SmallPtrSet<const MachineInstr *, 4> &BeforeSet,
+                     const SmallPtrSet<const MachineInstr *, 4> &AfterSet) {
+  auto InsertPos = MBB->end();
+  while (InsertPos != MBB->begin()) {
+    if (BeforeSet.count(&*std::prev(InsertPos))) {
+#ifndef NDEBUG
+      // Sanity check
+      for (auto Pos = InsertPos, E = MBB->begin(); Pos != E; --Pos)
+        assert(!AfterSet.count(&*std::prev(Pos)));
+#endif
+      break;
+    }
+    --InsertPos;
+  }
+  return InsertPos;
+}
+
+// Returns an iterator to the latest position possible within the MBB,
+// satisfying the restrictions given by BeforeSet and AfterSet. BeforeSet
+// contains instructions that should go before the marker, and AfterSet contains
+// ones that should go after the marker. In this function, BeforeSet is only
+// used for sanity checking.
+static MachineBasicBlock::iterator
+GetLatestInsertPos(MachineBasicBlock *MBB,
+                   const SmallPtrSet<const MachineInstr *, 4> &BeforeSet,
+                   const SmallPtrSet<const MachineInstr *, 4> &AfterSet) {
+  auto InsertPos = MBB->begin();
+  while (InsertPos != MBB->end()) {
+    if (AfterSet.count(&*InsertPos)) {
+#ifndef NDEBUG
+      // Sanity check
+      for (auto Pos = InsertPos, E = MBB->end(); Pos != E; ++Pos)
+        assert(!BeforeSet.count(&*Pos));
+#endif
+      break;
+    }
+    ++InsertPos;
+  }
+  return InsertPos;
+}
+
+void WebAssemblyCFGStackify::registerScope(MachineInstr *Begin,
+                                           MachineInstr *End) {
+  BeginToEnd[Begin] = End;
+  EndToBegin[End] = Begin;
+}
+
+void WebAssemblyCFGStackify::registerTryScope(MachineInstr *Begin,
+                                              MachineInstr *End,
+                                              MachineBasicBlock *EHPad) {
+  registerScope(Begin, End);
+  TryToEHPad[Begin] = EHPad;
+  EHPadToTry[EHPad] = Begin;
+}
+
+// Given a LOOP/TRY marker, returns its bottom BB. Use cached information if any
+// to prevent recomputation.
+MachineBasicBlock *
+WebAssemblyCFGStackify::getBottom(const MachineInstr *Begin) {
+  const auto &MLI = getAnalysis<MachineLoopInfo>();
+  const auto &WEI = getAnalysis<WebAssemblyExceptionInfo>();
+  if (BeginToBottom.count(Begin))
+    return BeginToBottom[Begin];
+  if (Begin->getOpcode() == WebAssembly::LOOP) {
+    MachineLoop *L = MLI.getLoopFor(Begin->getParent());
+    assert(L);
+    BeginToBottom[Begin] = WebAssembly::getBottom(L);
+  } else if (Begin->getOpcode() == WebAssembly::TRY) {
+    WebAssemblyException *WE = WEI.getExceptionFor(TryToEHPad[Begin]);
+    assert(WE);
+    BeginToBottom[Begin] = WebAssembly::getBottom(WE);
+  } else
+    assert(false);
+  return BeginToBottom[Begin];
+}
+
 /// Insert a BLOCK marker for branches to MBB (if needed).
-static void PlaceBlockMarker(
-    MachineBasicBlock &MBB, MachineFunction &MF,
-    SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
-    DenseMap<const MachineInstr *, MachineInstr *> &BlockTops,
-    DenseMap<const MachineInstr *, MachineInstr *> &LoopTops,
-    const WebAssemblyInstrInfo &TII,
-    const MachineLoopInfo &MLI,
-    MachineDominatorTree &MDT,
-    WebAssemblyFunctionInfo &MFI) {
+void WebAssemblyCFGStackify::placeBlockMarker(MachineBasicBlock &MBB) {
+  // This should have been handled in placeTryMarker.
+  if (MBB.isEHPad())
+    return;
+
+  MachineFunction &MF = *MBB.getParent();
+  auto &MDT = getAnalysis<MachineDominatorTree>();
+  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  const auto &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
+
   // First compute the nearest common dominator of all forward non-fallthrough
   // predecessors so that we minimize the time that the BLOCK is on the stack,
   // which reduces overall stack height.
   MachineBasicBlock *Header = nullptr;
   bool IsBranchedTo = false;
   int MBBNumber = MBB.getNumber();
-  for (MachineBasicBlock *Pred : MBB.predecessors())
+  for (MachineBasicBlock *Pred : MBB.predecessors()) {
     if (Pred->getNumber() < MBBNumber) {
       Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred;
       if (ExplicitlyBranchesTo(Pred, &MBB))
         IsBranchedTo = true;
     }
+  }
   if (!Header)
     return;
   if (!IsBranchedTo)
@@ -125,43 +250,93 @@ static void PlaceBlockMarker(
   }
 
   // Decide where in Header to put the BLOCK.
-  MachineBasicBlock::iterator InsertPos;
-  MachineLoop *HeaderLoop = MLI.getLoopFor(Header);
-  if (HeaderLoop &&
-      MBB.getNumber() > WebAssembly::getBottom(HeaderLoop)->getNumber()) {
-    // Header is the header of a loop that does not lexically contain MBB, so
-    // the BLOCK needs to be above the LOOP, after any END constructs.
-    InsertPos = Header->begin();
-    while (InsertPos->getOpcode() == WebAssembly::END_BLOCK ||
-           InsertPos->getOpcode() == WebAssembly::END_LOOP)
-      ++InsertPos;
-  } else {
-    // Otherwise, insert the BLOCK as late in Header as we can, but before the
-    // beginning of the local expression tree and any nested BLOCKs.
-    InsertPos = Header->getFirstTerminator();
-    while (InsertPos != Header->begin() &&
-           WebAssembly::isChild(*std::prev(InsertPos), MFI) &&
-           std::prev(InsertPos)->getOpcode() != WebAssembly::LOOP &&
-           std::prev(InsertPos)->getOpcode() != WebAssembly::END_BLOCK &&
-           std::prev(InsertPos)->getOpcode() != WebAssembly::END_LOOP)
-      --InsertPos;
+
+  // Instructions that should go before the BLOCK.
+  SmallPtrSet<const MachineInstr *, 4> BeforeSet;
+  // Instructions that should go after the BLOCK.
+  SmallPtrSet<const MachineInstr *, 4> AfterSet;
+  for (const auto &MI : *Header) {
+    // If there is a previously placed LOOP/TRY marker and the bottom block of
+    // the loop/exception is above MBB, it should be after the BLOCK, because
+    // the loop/exception is nested in this block. Otherwise it should be before
+    // the BLOCK.
+    if (MI.getOpcode() == WebAssembly::LOOP ||
+        MI.getOpcode() == WebAssembly::TRY) {
+      if (MBB.getNumber() > getBottom(&MI)->getNumber())
+        AfterSet.insert(&MI);
+#ifndef NDEBUG
+      else
+        BeforeSet.insert(&MI);
+#endif
+    }
+
+    // All previously inserted BLOCK markers should be after the BLOCK because
+    // they are all nested blocks.
+    if (MI.getOpcode() == WebAssembly::BLOCK)
+      AfterSet.insert(&MI);
+
+#ifndef NDEBUG
+    // All END_(BLOCK|LOOP|TRY) markers should be before the BLOCK.
+    if (MI.getOpcode() == WebAssembly::END_BLOCK ||
+        MI.getOpcode() == WebAssembly::END_LOOP ||
+        MI.getOpcode() == WebAssembly::END_TRY)
+      BeforeSet.insert(&MI);
+#endif
+
+    // Terminators should go after the BLOCK.
+    if (MI.isTerminator())
+      AfterSet.insert(&MI);
+  }
+
+  // Local expression tree should go after the BLOCK.
+  for (auto I = Header->getFirstTerminator(), E = Header->begin(); I != E;
+       --I) {
+    if (std::prev(I)->isDebugInstr() || std::prev(I)->isPosition())
+      continue;
+    if (WebAssembly::isChild(*std::prev(I), MFI))
+      AfterSet.insert(&*std::prev(I));
+    else
+      break;
   }
 
   // Add the BLOCK.
+  auto InsertPos = GetLatestInsertPos(Header, BeforeSet, AfterSet);
   MachineInstr *Begin =
       BuildMI(*Header, InsertPos, Header->findDebugLoc(InsertPos),
               TII.get(WebAssembly::BLOCK))
           .addImm(int64_t(WebAssembly::ExprType::Void));
 
+  // Decide where in Header to put the END_BLOCK.
+  BeforeSet.clear();
+  AfterSet.clear();
+  for (auto &MI : MBB) {
+#ifndef NDEBUG
+    // END_BLOCK should precede existing LOOP and TRY markers.
+    if (MI.getOpcode() == WebAssembly::LOOP ||
+        MI.getOpcode() == WebAssembly::TRY)
+      AfterSet.insert(&MI);
+#endif
+
+    // If there is a previously placed END_LOOP marker and the header of the
+    // loop is above this block's header, the END_LOOP should be placed after
+    // the BLOCK, because the loop contains this block. Otherwise the END_LOOP
+    // should be placed before the BLOCK. The same for END_TRY.
+    if (MI.getOpcode() == WebAssembly::END_LOOP ||
+        MI.getOpcode() == WebAssembly::END_TRY) {
+      if (EndToBegin[&MI]->getParent()->getNumber() >= Header->getNumber())
+        BeforeSet.insert(&MI);
+#ifndef NDEBUG
+      else
+        AfterSet.insert(&MI);
+#endif
+    }
+  }
+
   // Mark the end of the block.
-  InsertPos = MBB.begin();
-  while (InsertPos != MBB.end() &&
-         InsertPos->getOpcode() == WebAssembly::END_LOOP &&
-         LoopTops[&*InsertPos]->getParent()->getNumber() >= Header->getNumber())
-    ++InsertPos;
+  InsertPos = GetEarliestInsertPos(&MBB, BeforeSet, AfterSet);
   MachineInstr *End = BuildMI(MBB, InsertPos, MBB.findPrevDebugLoc(InsertPos),
                               TII.get(WebAssembly::END_BLOCK));
-  BlockTops[End] = Begin;
+  registerScope(Begin, End);
 
   // Track the farthest-spanning scope that ends at this point.
   int Number = MBB.getNumber();
@@ -171,11 +346,11 @@ static void PlaceBlockMarker(
 }
 
 /// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
-static void PlaceLoopMarker(
-    MachineBasicBlock &MBB, MachineFunction &MF,
-    SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
-    DenseMap<const MachineInstr *, MachineInstr *> &LoopTops,
-    const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI) {
+void WebAssemblyCFGStackify::placeLoopMarker(MachineBasicBlock &MBB) {
+  MachineFunction &MF = *MBB.getParent();
+  const auto &MLI = getAnalysis<MachineLoopInfo>();
+  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+
   MachineLoop *Loop = MLI.getLoopFor(&MBB);
   if (!Loop || Loop->getHeader() != &MBB)
     return;
@@ -193,22 +368,43 @@ static void PlaceLoopMarker(
   }
   MachineBasicBlock *AfterLoop = &*Iter;
 
-  // Mark the beginning of the loop (after the end of any existing loop that
-  // ends here).
-  auto InsertPos = MBB.begin();
-  while (InsertPos != MBB.end() &&
-         InsertPos->getOpcode() == WebAssembly::END_LOOP)
-    ++InsertPos;
+  // Decide where in Header to put the LOOP.
+  SmallPtrSet<const MachineInstr *, 4> BeforeSet;
+  SmallPtrSet<const MachineInstr *, 4> AfterSet;
+  for (const auto &MI : MBB) {
+    // LOOP marker should be after any existing loop that ends here. Otherwise
+    // we assume the instruction belongs to the loop.
+    if (MI.getOpcode() == WebAssembly::END_LOOP)
+      BeforeSet.insert(&MI);
+#ifndef NDEBUG
+    else
+      AfterSet.insert(&MI);
+#endif
+  }
+
+  // Mark the beginning of the loop.
+  auto InsertPos = GetEarliestInsertPos(&MBB, BeforeSet, AfterSet);
   MachineInstr *Begin = BuildMI(MBB, InsertPos, MBB.findDebugLoc(InsertPos),
                                 TII.get(WebAssembly::LOOP))
                             .addImm(int64_t(WebAssembly::ExprType::Void));
 
-  // Mark the end of the loop (using arbitrary debug location that branched
-  // to the loop end as its location).
+  // Decide where in Header to put the END_LOOP.
+  BeforeSet.clear();
+  AfterSet.clear();
+#ifndef NDEBUG
+  for (const auto &MI : MBB)
+    // Existing END_LOOP markers belong to parent loops of this loop
+    if (MI.getOpcode() == WebAssembly::END_LOOP)
+      AfterSet.insert(&MI);
+#endif
+
+  // Mark the end of the loop (using arbitrary debug location that branched to
+  // the loop end as its location).
+  InsertPos = GetEarliestInsertPos(AfterLoop, BeforeSet, AfterSet);
   DebugLoc EndDL = (*AfterLoop->pred_rbegin())->findBranchDebugLoc();
-  MachineInstr *End = BuildMI(*AfterLoop, AfterLoop->begin(), EndDL,
-                              TII.get(WebAssembly::END_LOOP));
-  LoopTops[End] = Begin;
+  MachineInstr *End =
+      BuildMI(*AfterLoop, InsertPos, EndDL, TII.get(WebAssembly::END_LOOP));
+  registerScope(Begin, End);
 
   assert((!ScopeTops[AfterLoop->getNumber()] ||
           ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
@@ -217,6 +413,183 @@ static void PlaceLoopMarker(
     ScopeTops[AfterLoop->getNumber()] = &MBB;
 }
 
+void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
+  if (!MBB.isEHPad())
+    return;
+
+  // catch_all terminate pad is grouped together with catch terminate pad and
+  // does not need a separate TRY and END_TRY marker.
+  if (WebAssembly::isCatchAllTerminatePad(MBB))
+    return;
+
+  MachineFunction &MF = *MBB.getParent();
+  auto &MDT = getAnalysis<MachineDominatorTree>();
+  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  const auto &WEI = getAnalysis<WebAssemblyExceptionInfo>();
+  const auto &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
+
+  // Compute the nearest common dominator of all unwind predecessors
+  MachineBasicBlock *Header = nullptr;
+  int MBBNumber = MBB.getNumber();
+  for (auto *Pred : MBB.predecessors()) {
+    if (Pred->getNumber() < MBBNumber) {
+      Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred;
+      assert(!ExplicitlyBranchesTo(Pred, &MBB) &&
+             "Explicit branch to an EH pad!");
+    }
+  }
+  if (!Header)
+    return;
+
+  // If this try is at the bottom of the function, insert a dummy block at the
+  // end.
+  WebAssemblyException *WE = WEI.getExceptionFor(&MBB);
+  assert(WE);
+  MachineBasicBlock *Bottom = WebAssembly::getBottom(WE);
+
+  auto Iter = std::next(MachineFunction::iterator(Bottom));
+  if (Iter == MF.end()) {
+    MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
+    // Give it a fake predecessor so that AsmPrinter prints its label.
+    Label->addSuccessor(Label);
+    MF.push_back(Label);
+    Iter = std::next(MachineFunction::iterator(Bottom));
+  }
+  MachineBasicBlock *AfterTry = &*Iter;
+
+  assert(AfterTry != &MF.front());
+  MachineBasicBlock *LayoutPred =
+      &*std::prev(MachineFunction::iterator(AfterTry));
+
+  // If the nearest common dominator is inside a more deeply nested context,
+  // walk out to the nearest scope which isn't more deeply nested.
+  for (MachineFunction::iterator I(LayoutPred), E(Header); I != E; --I) {
+    if (MachineBasicBlock *ScopeTop = ScopeTops[I->getNumber()]) {
+      if (ScopeTop->getNumber() > Header->getNumber()) {
+        // Skip over an intervening scope.
+        I = std::next(MachineFunction::iterator(ScopeTop));
+      } else {
+        // We found a scope level at an appropriate depth.
+        Header = ScopeTop;
+        break;
+      }
+    }
+  }
+
+  // Decide where in Header to put the TRY.
+
+  // Instructions that should go before the BLOCK.
+  SmallPtrSet<const MachineInstr *, 4> BeforeSet;
+  // Instructions that should go after the BLOCK.
+  SmallPtrSet<const MachineInstr *, 4> AfterSet;
+  for (const auto &MI : *Header) {
+    // If there is a previously placed LOOP marker and the bottom block of
+    // the loop is above MBB, the LOOP should be after the TRY, because the
+    // loop is nested in this try. Otherwise it should be before the TRY.
+    if (MI.getOpcode() == WebAssembly::LOOP) {
+      if (MBB.getNumber() > Bottom->getNumber())
+        AfterSet.insert(&MI);
+#ifndef NDEBUG
+      else
+        BeforeSet.insert(&MI);
+#endif
+    }
+
+    // All previously inserted TRY markers should be after the TRY because they
+    // are all nested trys.
+    if (MI.getOpcode() == WebAssembly::TRY)
+      AfterSet.insert(&MI);
+
+#ifndef NDEBUG
+    // All END_(LOOP/TRY) markers should be before the TRY.
+    if (MI.getOpcode() == WebAssembly::END_LOOP ||
+        MI.getOpcode() == WebAssembly::END_TRY)
+      BeforeSet.insert(&MI);
+#endif
+
+    // Terminators should go after the TRY.
+    if (MI.isTerminator())
+      AfterSet.insert(&MI);
+  }
+
+  // Local expression tree should go after the TRY.
+  for (auto I = Header->getFirstTerminator(), E = Header->begin(); I != E;
+       --I) {
+    if (std::prev(I)->isDebugInstr() || std::prev(I)->isPosition())
+      continue;
+    if (WebAssembly::isChild(*std::prev(I), MFI))
+      AfterSet.insert(&*std::prev(I));
+    else
+      break;
+  }
+
+  // If Header unwinds to MBB (= Header contains 'invoke'), the try block should
+  // contain the call within it. So the call should go after the TRY. The
+  // exception is when the header's terminator is a rethrow instruction, in
+  // which case that instruction, not a call instruction before it, is gonna
+  // throw.
+  if (MBB.isPredecessor(Header)) {
+    auto TermPos = Header->getFirstTerminator();
+    if (TermPos == Header->end() || !WebAssembly::isRethrow(*TermPos)) {
+      for (const auto &MI : reverse(*Header)) {
+        if (MI.isCall()) {
+          AfterSet.insert(&MI);
+          break;
+        }
+      }
+    }
+  }
+
+  // Add the TRY.
+  auto InsertPos = GetLatestInsertPos(Header, BeforeSet, AfterSet);
+  MachineInstr *Begin =
+      BuildMI(*Header, InsertPos, Header->findDebugLoc(InsertPos),
+              TII.get(WebAssembly::TRY))
+          .addImm(int64_t(WebAssembly::ExprType::Void));
+
+  // Decide where in Header to put the END_TRY.
+  BeforeSet.clear();
+  AfterSet.clear();
+  for (const auto &MI : *AfterTry) {
+#ifndef NDEBUG
+    // END_TRY should precede existing LOOP markers.
+    if (MI.getOpcode() == WebAssembly::LOOP)
+      AfterSet.insert(&MI);
+
+    // All END_TRY markers placed earlier belong to exceptions that contains
+    // this one.
+    if (MI.getOpcode() == WebAssembly::END_TRY)
+      AfterSet.insert(&MI);
+#endif
+
+    // If there is a previously placed END_LOOP marker and its header is after
+    // where TRY marker is, this loop is contained within the 'catch' part, so
+    // the END_TRY marker should go after that. Otherwise, the whole try-catch
+    // is contained within this loop, so the END_TRY should go before that.
+    if (MI.getOpcode() == WebAssembly::END_LOOP) {
+      if (EndToBegin[&MI]->getParent()->getNumber() >= Header->getNumber())
+        BeforeSet.insert(&MI);
+#ifndef NDEBUG
+      else
+        AfterSet.insert(&MI);
+#endif
+    }
+  }
+
+  // Mark the end of the TRY.
+  InsertPos = GetEarliestInsertPos(AfterTry, BeforeSet, AfterSet);
+  MachineInstr *End =
+      BuildMI(*AfterTry, InsertPos, Bottom->findBranchDebugLoc(),
+              TII.get(WebAssembly::END_TRY));
+  registerTryScope(Begin, End, &MBB);
+
+  // Track the farthest-spanning scope that ends at this point.
+  int Number = AfterTry->getNumber();
+  if (!ScopeTops[Number] ||
+      ScopeTops[Number]->getNumber() > Header->getNumber())
+    ScopeTops[Number] = Header;
+}
+
 static unsigned
 GetDepth(const SmallVectorImpl<const MachineBasicBlock *> &Stack,
          const MachineBasicBlock *MBB) {
@@ -237,11 +610,8 @@ GetDepth(const SmallVectorImpl<const MachineBasicBlock *> &Stack,
 /// that end at the function end need to have a return type signature that
 /// matches the function signature, even though it's unreachable. This function
 /// checks for such cases and fixes up the signatures.
-static void FixEndsAtEndOfFunction(
-    MachineFunction &MF,
-    const WebAssemblyFunctionInfo &MFI,
-    DenseMap<const MachineInstr *, MachineInstr *> &BlockTops,
-    DenseMap<const MachineInstr *, MachineInstr *> &LoopTops) {
+void WebAssemblyCFGStackify::fixEndsAtEndOfFunction(MachineFunction &MF) {
+  const auto &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
   assert(MFI.getResults().size() <= 1);
 
   if (MFI.getResults().empty())
@@ -249,16 +619,31 @@ static void FixEndsAtEndOfFunction(
 
   WebAssembly::ExprType retType;
   switch (MFI.getResults().front().SimpleTy) {
-  case MVT::i32: retType = WebAssembly::ExprType::I32; break;
-  case MVT::i64: retType = WebAssembly::ExprType::I64; break;
-  case MVT::f32: retType = WebAssembly::ExprType::F32; break;
-  case MVT::f64: retType = WebAssembly::ExprType::F64; break;
-  case MVT::v16i8: retType = WebAssembly::ExprType::I8x16; break;
-  case MVT::v8i16: retType = WebAssembly::ExprType::I16x8; break;
-  case MVT::v4i32: retType = WebAssembly::ExprType::I32x4; break;
-  case MVT::v4f32: retType = WebAssembly::ExprType::F32x4; break;
-  case MVT::ExceptRef: retType = WebAssembly::ExprType::ExceptRef; break;
-  default: llvm_unreachable("unexpected return type");
+  case MVT::i32:
+    retType = WebAssembly::ExprType::I32;
+    break;
+  case MVT::i64:
+    retType = WebAssembly::ExprType::I64;
+    break;
+  case MVT::f32:
+    retType = WebAssembly::ExprType::F32;
+    break;
+  case MVT::f64:
+    retType = WebAssembly::ExprType::F64;
+    break;
+  case MVT::v16i8:
+  case MVT::v8i16:
+  case MVT::v4i32:
+  case MVT::v2i64:
+  case MVT::v4f32:
+  case MVT::v2f64:
+    retType = WebAssembly::ExprType::V128;
+    break;
+  case MVT::ExceptRef:
+    retType = WebAssembly::ExprType::ExceptRef;
+    break;
+  default:
+    llvm_unreachable("unexpected return type");
   }
 
   for (MachineBasicBlock &MBB : reverse(MF)) {
@@ -266,11 +651,11 @@ static void FixEndsAtEndOfFunction(
       if (MI.isPosition() || MI.isDebugInstr())
         continue;
       if (MI.getOpcode() == WebAssembly::END_BLOCK) {
-        BlockTops[&MI]->getOperand(0).setImm(int32_t(retType));
+        EndToBegin[&MI]->getOperand(0).setImm(int32_t(retType));
         continue;
       }
       if (MI.getOpcode() == WebAssembly::END_LOOP) {
-        LoopTops[&MI]->getOperand(0).setImm(int32_t(retType));
+        EndToBegin[&MI]->getOperand(0).setImm(int32_t(retType));
         continue;
       }
       // Something other than an `end`. We're done.
@@ -281,60 +666,108 @@ static void FixEndsAtEndOfFunction(
 
 // WebAssembly functions end with an end instruction, as if the function body
 // were a block.
-static void AppendEndToFunction(
-    MachineFunction &MF,
-    const WebAssemblyInstrInfo &TII) {
+static void AppendEndToFunction(MachineFunction &MF,
+                                const WebAssemblyInstrInfo &TII) {
   BuildMI(MF.back(), MF.back().end(),
           MF.back().findPrevDebugLoc(MF.back().end()),
           TII.get(WebAssembly::END_FUNCTION));
 }
 
-/// Insert LOOP and BLOCK markers at appropriate places.
-static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI,
-                         const WebAssemblyInstrInfo &TII,
-                         MachineDominatorTree &MDT,
-                         WebAssemblyFunctionInfo &MFI) {
-  // For each block whose label represents the end of a scope, record the block
-  // which holds the beginning of the scope. This will allow us to quickly skip
-  // over scoped regions when walking blocks. We allocate one more than the
-  // number of blocks in the function to accommodate for the possible fake block
-  // we may insert at the end.
-  SmallVector<MachineBasicBlock *, 8> ScopeTops(MF.getNumBlockIDs() + 1);
-
-  // For each LOOP_END, the corresponding LOOP.
-  DenseMap<const MachineInstr *, MachineInstr *> LoopTops;
-
-  // For each END_BLOCK, the corresponding BLOCK.
-  DenseMap<const MachineInstr *, MachineInstr *> BlockTops;
-
-  for (auto &MBB : MF) {
-    // Place the LOOP for MBB if MBB is the header of a loop.
-    PlaceLoopMarker(MBB, MF, ScopeTops, LoopTops, TII, MLI);
-
-    // Place the BLOCK for MBB if MBB is branched to from above.
-    PlaceBlockMarker(MBB, MF, ScopeTops, BlockTops, LoopTops, TII, MLI, MDT, MFI);
-  }
+/// Insert LOOP/TRY/BLOCK markers at appropriate places.
+void WebAssemblyCFGStackify::placeMarkers(MachineFunction &MF) {
+  const MCAsmInfo *MCAI = MF.getTarget().getMCAsmInfo();
+  // We allocate one more than the number of blocks in the function to
+  // accommodate for the possible fake block we may insert at the end.
+  ScopeTops.resize(MF.getNumBlockIDs() + 1);
+  // Place the LOOP for MBB if MBB is the header of a loop.
+  for (auto &MBB : MF)
+    placeLoopMarker(MBB);
+  // Place the TRY for MBB if MBB is the EH pad of an exception.
+  if (MCAI->getExceptionHandlingType() == ExceptionHandling::Wasm &&
+      MF.getFunction().hasPersonalityFn())
+    for (auto &MBB : MF)
+      placeTryMarker(MBB);
+  // Place the BLOCK for MBB if MBB is branched to from above.
+  for (auto &MBB : MF)
+    placeBlockMarker(MBB);
+}
 
+void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) {
+  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   // Now rewrite references to basic blocks to be depth immediates.
+  // We need two stacks: one for normal scopes and the other for EH pad scopes.
+  // EH pad stack is used to rewrite depths in rethrow instructions.
   SmallVector<const MachineBasicBlock *, 8> Stack;
+  SmallVector<const MachineBasicBlock *, 8> EHPadStack;
   for (auto &MBB : reverse(MF)) {
-    for (auto &MI : reverse(MBB)) {
+    for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
+      MachineInstr &MI = *I;
       switch (MI.getOpcode()) {
       case WebAssembly::BLOCK:
-        assert(ScopeTops[Stack.back()->getNumber()]->getNumber() <= MBB.getNumber() &&
-               "Block should be balanced");
+        assert(ScopeTops[Stack.back()->getNumber()]->getNumber() <=
+                   MBB.getNumber() &&
+               "Block/try should be balanced");
         Stack.pop_back();
         break;
+
+      case WebAssembly::TRY:
+        assert(ScopeTops[Stack.back()->getNumber()]->getNumber() <=
+                   MBB.getNumber() &&
+               "Block/try marker should be balanced");
+        Stack.pop_back();
+        EHPadStack.pop_back();
+        break;
+
+      case WebAssembly::CATCH_I32:
+      case WebAssembly::CATCH_I64:
+      case WebAssembly::CATCH_ALL:
+        // Currently the only case there are more than one catch for a try is
+        // for catch terminate pad, in the form of
+        //   try
+        //   catch
+        //     call @__clang_call_terminate
+        //     unreachable
+        //   catch_all
+        //     call @std::terminate
+        //     unreachable
+        //   end
+        // So we shouldn't push the current BB for the second catch_all block
+        // here.
+        if (!WebAssembly::isCatchAllTerminatePad(MBB))
+          EHPadStack.push_back(&MBB);
+        break;
+
       case WebAssembly::LOOP:
         assert(Stack.back() == &MBB && "Loop top should be balanced");
         Stack.pop_back();
         break;
+
       case WebAssembly::END_BLOCK:
+      case WebAssembly::END_TRY:
         Stack.push_back(&MBB);
         break;
+
       case WebAssembly::END_LOOP:
-        Stack.push_back(LoopTops[&MI]->getParent());
+        Stack.push_back(EndToBegin[&MI]->getParent());
+        break;
+
+      case WebAssembly::RETHROW: {
+        // Rewrite MBB operands to be depth immediates.
+        unsigned EHPadDepth = GetDepth(EHPadStack, MI.getOperand(0).getMBB());
+        MI.RemoveOperand(0);
+        MI.addOperand(MF, MachineOperand::CreateImm(EHPadDepth));
         break;
+      }
+
+      case WebAssembly::RETHROW_TO_CALLER: {
+        MachineInstr *Rethrow =
+            BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(WebAssembly::RETHROW))
+                .addImm(EHPadStack.size());
+        MI.eraseFromParent();
+        I = MachineBasicBlock::reverse_iterator(Rethrow);
+        break;
+      }
+
       default:
         if (MI.isTerminator()) {
           // Rewrite MBB operands to be depth immediates.
@@ -352,13 +785,15 @@ static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI,
     }
   }
   assert(Stack.empty() && "Control flow should be balanced");
+}
 
-  // Fix up block/loop signatures at the end of the function to conform to
-  // WebAssembly's rules.
-  FixEndsAtEndOfFunction(MF, MFI, BlockTops, LoopTops);
-
-  // Add an end instruction at the end of the function body.
-  AppendEndToFunction(MF, TII);
+void WebAssemblyCFGStackify::releaseMemory() {
+  ScopeTops.clear();
+  BeginToEnd.clear();
+  EndToBegin.clear();
+  TryToEHPad.clear();
+  EHPadToTry.clear();
+  BeginToBottom.clear();
 }
 
 bool WebAssemblyCFGStackify::runOnMachineFunction(MachineFunction &MF) {
@@ -366,15 +801,27 @@ bool WebAssemblyCFGStackify::runOnMachineFunction(MachineFunction &MF) {
                        "********** Function: "
                     << MF.getName() << '\n');
 
-  const auto &MLI = getAnalysis<MachineLoopInfo>();
-  auto &MDT = getAnalysis<MachineDominatorTree>();
+  releaseMemory();
+
   // Liveness is not tracked for VALUE_STACK physreg.
-  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
-  WebAssemblyFunctionInfo &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
   MF.getRegInfo().invalidateLiveness();
 
-  // Place the BLOCK and LOOP markers to indicate the beginnings of scopes.
-  PlaceMarkers(MF, MLI, TII, MDT, MFI);
+  // Place the BLOCK/LOOP/TRY markers to indicate the beginnings of scopes.
+  placeMarkers(MF);
+
+  // Convert MBB operands in terminators to relative depth immediates.
+  rewriteDepthImmediates(MF);
+
+  // Fix up block/loop/try signatures at the end of the function to conform to
+  // WebAssembly's rules.
+  fixEndsAtEndOfFunction(MF);
+
+  // Add an end instruction at the end of the function body.
+  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  if (!MF.getSubtarget<WebAssemblySubtarget>()
+           .getTargetTriple()
+           .isOSBinFormatELF())
+    AppendEndToFunction(MF, TII);
 
   return true;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp b/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp
index c1820bf66bc0..aaa6d286598f 100644
--- a/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp
@@ -64,16 +64,30 @@ FunctionPass *llvm::createWebAssemblyCallIndirectFixup() {
 static unsigned GetNonPseudoCallIndirectOpcode(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
     using namespace WebAssembly;
-  case PCALL_INDIRECT_VOID: return CALL_INDIRECT_VOID;
-  case PCALL_INDIRECT_I32: return CALL_INDIRECT_I32;
-  case PCALL_INDIRECT_I64: return CALL_INDIRECT_I64;
-  case PCALL_INDIRECT_F32: return CALL_INDIRECT_F32;
-  case PCALL_INDIRECT_F64: return CALL_INDIRECT_F64;
-  case PCALL_INDIRECT_v16i8: return CALL_INDIRECT_v16i8;
-  case PCALL_INDIRECT_v8i16: return CALL_INDIRECT_v8i16;
-  case PCALL_INDIRECT_v4i32: return CALL_INDIRECT_v4i32;
-  case PCALL_INDIRECT_v4f32: return CALL_INDIRECT_v4f32;
-  default: return INSTRUCTION_LIST_END;
+  case PCALL_INDIRECT_VOID:
+    return CALL_INDIRECT_VOID;
+  case PCALL_INDIRECT_I32:
+    return CALL_INDIRECT_I32;
+  case PCALL_INDIRECT_I64:
+    return CALL_INDIRECT_I64;
+  case PCALL_INDIRECT_F32:
+    return CALL_INDIRECT_F32;
+  case PCALL_INDIRECT_F64:
+    return CALL_INDIRECT_F64;
+  case PCALL_INDIRECT_v16i8:
+    return CALL_INDIRECT_v16i8;
+  case PCALL_INDIRECT_v8i16:
+    return CALL_INDIRECT_v8i16;
+  case PCALL_INDIRECT_v4i32:
+    return CALL_INDIRECT_v4i32;
+  case PCALL_INDIRECT_v2i64:
+    return CALL_INDIRECT_v2i64;
+  case PCALL_INDIRECT_v4f32:
+    return CALL_INDIRECT_v4f32;
+  case PCALL_INDIRECT_v2f64:
+    return CALL_INDIRECT_v2f64;
+  default:
+    return INSTRUCTION_LIST_END;
   }
 }
 
@@ -84,7 +98,7 @@ static bool IsPseudoCallIndirect(const MachineInstr &MI) {
 
 bool WebAssemblyCallIndirectFixup::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "********** Fixing up CALL_INDIRECTs **********\n"
-                    << MF.getName() << '\n');
+                    << "********** Function: " << MF.getName() << '\n');
 
   bool Changed = false;
   const WebAssemblyInstrInfo *TII =
@@ -110,10 +124,8 @@ bool WebAssemblyCallIndirectFixup::runOnMachineFunction(MachineFunction &MF) {
         Ops.push_back(MachineOperand::CreateImm(0));
 
         for (const MachineOperand &MO :
-                 make_range(MI.operands_begin() +
-                                MI.getDesc().getNumDefs() + 1,
-                            MI.operands_begin() +
-                                MI.getNumExplicitOperands()))
+             make_range(MI.operands_begin() + MI.getDesc().getNumDefs() + 1,
+                        MI.operands_begin() + MI.getNumExplicitOperands()))
           Ops.push_back(MO);
         Ops.push_back(MI.getOperand(MI.getDesc().getNumDefs()));
 
@@ -133,4 +145,3 @@ bool WebAssemblyCallIndirectFixup::runOnMachineFunction(MachineFunction &MF) {
 
   return Changed;
 }
-
diff --git a/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
new file mode 100644
index 000000000000..8ecc159951ad
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
@@ -0,0 +1,46 @@
+//===-- WebAssemblyDebugValueManager.cpp - WebAssembly DebugValue Manager -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the manager for MachineInstr DebugValues.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyDebugValueManager.h"
+#include "WebAssemblyMachineFunctionInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+
+using namespace llvm;
+
+WebAssemblyDebugValueManager::WebAssemblyDebugValueManager(
+    MachineInstr *Instr) {
+  Instr->collectDebugValues(DbgValues);
+}
+
+void WebAssemblyDebugValueManager::move(MachineInstr *Insert) {
+  MachineBasicBlock *MBB = Insert->getParent();
+  for (MachineInstr *DBI : reverse(DbgValues))
+    MBB->splice(Insert, DBI->getParent(), DBI);
+}
+
+void WebAssemblyDebugValueManager::updateReg(unsigned Reg) {
+  for (auto *DBI : DbgValues)
+    DBI->getOperand(0).setReg(Reg);
+}
+
+void WebAssemblyDebugValueManager::clone(MachineInstr *Insert,
+                                         unsigned NewReg) {
+  MachineBasicBlock *MBB = Insert->getParent();
+  MachineFunction *MF = MBB->getParent();
+  for (MachineInstr *DBI : reverse(DbgValues)) {
+    MachineInstr *Clone = MF->CloneMachineInstr(DBI);
+    Clone->getOperand(0).setReg(NewReg);
+    MBB->insert(Insert, Clone);
+  }
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h
new file mode 100644
index 000000000000..73f317214058
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h
@@ -0,0 +1,38 @@
+// WebAssemblyDebugValueManager.h - WebAssembly DebugValue Manager -*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declaration of the WebAssembly-specific
+/// manager for DebugValues associated with the specific MachineInstr.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYDEBUGVALUEMANAGER_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYDEBUGVALUEMANAGER_H
+
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+class MachineInstr;
+
+class WebAssemblyDebugValueManager {
+  SmallVector<MachineInstr *, 2> DbgValues;
+
+public:
+  WebAssemblyDebugValueManager(MachineInstr *Instr);
+
+  void move(MachineInstr *Insert);
+  void updateReg(unsigned Reg);
+  void clone(MachineInstr *Insert, unsigned NewReg);
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp b/lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp
new file mode 100644
index 000000000000..c86260ba408c
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp
@@ -0,0 +1,87 @@
+//===-- WebAssemblyEHRestoreStackPointer.cpp - __stack_pointer restoration ===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// After the stack is unwound due to a thrown exception, the __stack_pointer
+/// global can point to an invalid address. This inserts instructions that
+/// restore __stack_pointer global.
+///
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssembly.h"
+#include "WebAssemblySubtarget.h"
+#include "WebAssemblyUtilities.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "wasm-eh-restore-stack-pointer"
+
+namespace {
+class WebAssemblyEHRestoreStackPointer final : public MachineFunctionPass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+  WebAssemblyEHRestoreStackPointer() : MachineFunctionPass(ID) {}
+
+  StringRef getPassName() const override {
+    return "WebAssembly Restore Stack Pointer for Exception Handling";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+};
+} // end anonymous namespace
+
+char WebAssemblyEHRestoreStackPointer::ID = 0;
+INITIALIZE_PASS(WebAssemblyEHRestoreStackPointer, DEBUG_TYPE,
+                "Restore Stack Pointer for Exception Handling", true, false)
+
+FunctionPass *llvm::createWebAssemblyEHRestoreStackPointer() {
+  return new WebAssemblyEHRestoreStackPointer();
+}
+
+bool WebAssemblyEHRestoreStackPointer::runOnMachineFunction(
+    MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "********** EH Restore Stack Pointer **********\n"
+                       "********** Function: "
+                    << MF.getName() << '\n');
+
+  const auto *FrameLowering = static_cast<const WebAssemblyFrameLowering *>(
+      MF.getSubtarget().getFrameLowering());
+  if (!FrameLowering->needsPrologForEH(MF))
+    return false;
+  bool Changed = false;
+
+  for (auto &MBB : MF) {
+    if (!MBB.isEHPad())
+      continue;
+    Changed = true;
+
+    // Insert __stack_pointer restoring instructions at the beginning of each EH
+    // pad, after the catch instruction. (Catch instructions may have been
+    // reordered, and catch_all instructions have not been inserted yet, but
+    // those cases are handled in LateEHPrepare).
+    //
+    // Here it is safe to assume that SP32 holds the latest value of
+    // __stack_pointer, because the only exception for this case is when a
+    // function uses the red zone, but that only happens with leaf functions,
+    // and we don't restore __stack_pointer in leaf functions anyway.
+    auto InsertPos = MBB.begin();
+    if (WebAssembly::isCatch(*MBB.begin()))
+      InsertPos++;
+    FrameLowering->writeSPToGlobal(WebAssembly::SP32, MF, MBB, InsertPos,
+                                   MBB.begin()->getDebugLoc());
+  }
+  return Changed;
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp b/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp
index 84683d48a90a..6b3a3e765786 100644
--- a/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp
@@ -13,8 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyExceptionInfo.h"
-#include "WebAssemblyUtilities.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "WebAssemblyUtilities.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/CodeGen/MachineDominanceFrontier.h"
 #include "llvm/CodeGen/MachineDominators.h"
@@ -32,7 +32,10 @@ INITIALIZE_PASS_DEPENDENCY(MachineDominanceFrontier)
 INITIALIZE_PASS_END(WebAssemblyExceptionInfo, DEBUG_TYPE,
                     "WebAssembly Exception Information", true, true)
 
-bool WebAssemblyExceptionInfo::runOnMachineFunction(MachineFunction &F) {
+bool WebAssemblyExceptionInfo::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "********** Exception Info Calculation **********\n"
+                       "********** Function: "
+                    << MF.getName() << '\n');
   releaseMemory();
   auto &MDT = getAnalysis<MachineDominatorTree>();
   auto &MDF = getAnalysis<MachineDominanceFrontier>();
diff --git a/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
index 8619cbdcb5ee..27aabe6ba0bd 100644
--- a/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
@@ -11,7 +11,7 @@
 /// This file converts any remaining registers into WebAssembly locals.
 ///
 /// After register stackification and register coloring, convert non-stackified
-/// registers into locals, inserting explicit get_local and set_local
+/// registers into locals, inserting explicit local.get and local.set
 /// instructions.
 ///
 //===----------------------------------------------------------------------===//
@@ -31,12 +31,14 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-explicit-locals"
 
-// A command-line option to disable this pass. Note that this produces output
-// which is not valid WebAssembly, though it may be more convenient for writing
-// LLVM unit tests with.
-static cl::opt<bool> DisableWebAssemblyExplicitLocals(
-    "disable-wasm-explicit-locals", cl::ReallyHidden,
-    cl::desc("WebAssembly: Disable emission of get_local/set_local."),
+// A command-line option to disable this pass, and keep implicit locals
+// for the purpose of testing with lit/llc ONLY.
+// This produces output which is not valid WebAssembly, and is not supported
+// by assemblers/disassemblers and other MC based tools.
+static cl::opt<bool> WasmDisableExplicitLocals(
+    "wasm-disable-explicit-locals", cl::Hidden,
+    cl::desc("WebAssembly: output implicit locals in"
+             " instruction output for test purposes only."),
     cl::init(false));
 
 namespace {
@@ -94,54 +96,54 @@ static unsigned getDropOpcode(const TargetRegisterClass *RC) {
   llvm_unreachable("Unexpected register class");
 }
 
-/// Get the appropriate get_local opcode for the given register class.
+/// Get the appropriate local.get opcode for the given register class.
 static unsigned getGetLocalOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
-    return WebAssembly::GET_LOCAL_I32;
+    return WebAssembly::LOCAL_GET_I32;
   if (RC == &WebAssembly::I64RegClass)
-    return WebAssembly::GET_LOCAL_I64;
+    return WebAssembly::LOCAL_GET_I64;
   if (RC == &WebAssembly::F32RegClass)
-    return WebAssembly::GET_LOCAL_F32;
+    return WebAssembly::LOCAL_GET_F32;
   if (RC == &WebAssembly::F64RegClass)
-    return WebAssembly::GET_LOCAL_F64;
+    return WebAssembly::LOCAL_GET_F64;
   if (RC == &WebAssembly::V128RegClass)
-    return WebAssembly::GET_LOCAL_V128;
+    return WebAssembly::LOCAL_GET_V128;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
-    return WebAssembly::GET_LOCAL_EXCEPT_REF;
+    return WebAssembly::LOCAL_GET_EXCEPT_REF;
   llvm_unreachable("Unexpected register class");
 }
 
-/// Get the appropriate set_local opcode for the given register class.
+/// Get the appropriate local.set opcode for the given register class.
 static unsigned getSetLocalOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
-    return WebAssembly::SET_LOCAL_I32;
+    return WebAssembly::LOCAL_SET_I32;
   if (RC == &WebAssembly::I64RegClass)
-    return WebAssembly::SET_LOCAL_I64;
+    return WebAssembly::LOCAL_SET_I64;
   if (RC == &WebAssembly::F32RegClass)
-    return WebAssembly::SET_LOCAL_F32;
+    return WebAssembly::LOCAL_SET_F32;
   if (RC == &WebAssembly::F64RegClass)
-    return WebAssembly::SET_LOCAL_F64;
+    return WebAssembly::LOCAL_SET_F64;
   if (RC == &WebAssembly::V128RegClass)
-    return WebAssembly::SET_LOCAL_V128;
+    return WebAssembly::LOCAL_SET_V128;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
-    return WebAssembly::SET_LOCAL_EXCEPT_REF;
+    return WebAssembly::LOCAL_SET_EXCEPT_REF;
   llvm_unreachable("Unexpected register class");
 }
 
-/// Get the appropriate tee_local opcode for the given register class.
+/// Get the appropriate local.tee opcode for the given register class.
 static unsigned getTeeLocalOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
-    return WebAssembly::TEE_LOCAL_I32;
+    return WebAssembly::LOCAL_TEE_I32;
   if (RC == &WebAssembly::I64RegClass)
-    return WebAssembly::TEE_LOCAL_I64;
+    return WebAssembly::LOCAL_TEE_I64;
   if (RC == &WebAssembly::F32RegClass)
-    return WebAssembly::TEE_LOCAL_F32;
+    return WebAssembly::LOCAL_TEE_F32;
   if (RC == &WebAssembly::F64RegClass)
-    return WebAssembly::TEE_LOCAL_F64;
+    return WebAssembly::LOCAL_TEE_F64;
   if (RC == &WebAssembly::V128RegClass)
-    return WebAssembly::TEE_LOCAL_V128;
+    return WebAssembly::LOCAL_TEE_V128;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
-    return WebAssembly::TEE_LOCAL_EXCEPT_REF;
+    return WebAssembly::LOCAL_TEE_EXCEPT_REF;
   llvm_unreachable("Unexpected register class");
 }
 
@@ -155,6 +157,8 @@ static MVT typeForRegClass(const TargetRegisterClass *RC) {
     return MVT::f32;
   if (RC == &WebAssembly::F64RegClass)
     return MVT::f64;
+  if (RC == &WebAssembly::V128RegClass)
+    return MVT::v16i8;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
     return MVT::ExceptRef;
   llvm_unreachable("unrecognized register class");
@@ -162,7 +166,7 @@ static MVT typeForRegClass(const TargetRegisterClass *RC) {
 
 /// Given a MachineOperand of a stackified vreg, return the instruction at the
 /// start of the expression tree.
-static MachineInstr *FindStartOfTree(MachineOperand &MO,
+static MachineInstr *findStartOfTree(MachineOperand &MO,
                                      MachineRegisterInfo &MRI,
                                      WebAssemblyFunctionInfo &MFI) {
   unsigned Reg = MO.getReg();
@@ -173,7 +177,7 @@ static MachineInstr *FindStartOfTree(MachineOperand &MO,
   for (MachineOperand &DefMO : Def->explicit_uses()) {
     if (!DefMO.isReg())
       continue;
-    return FindStartOfTree(DefMO, MRI, MFI);
+    return findStartOfTree(DefMO, MRI, MFI);
   }
 
   // If there were no stackified uses, we've reached the start.
@@ -186,7 +190,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
                     << MF.getName() << '\n');
 
   // Disable this pass if directed to do so.
-  if (DisableWebAssemblyExplicitLocals)
+  if (WasmDisableExplicitLocals)
     return false;
 
   bool Changed = false;
@@ -206,19 +210,19 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
       break;
     unsigned Reg = MI.getOperand(0).getReg();
     assert(!MFI.isVRegStackified(Reg));
-    Reg2Local[Reg] = MI.getOperand(1).getImm();
+    Reg2Local[Reg] = static_cast<unsigned>(MI.getOperand(1).getImm());
     MI.eraseFromParent();
     Changed = true;
   }
 
   // Start assigning local numbers after the last parameter.
-  unsigned CurLocal = MFI.getParams().size();
+  unsigned CurLocal = static_cast<unsigned>(MFI.getParams().size());
 
   // Precompute the set of registers that are unused, so that we can insert
   // drops to their defs.
   BitVector UseEmpty(MRI.getNumVirtRegs());
-  for (unsigned i = 0, e = MRI.getNumVirtRegs(); i < e; ++i)
-    UseEmpty[i] = MRI.use_empty(TargetRegisterInfo::index2VirtReg(i));
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I < E; ++I)
+    UseEmpty[I] = MRI.use_empty(TargetRegisterInfo::index2VirtReg(I));
 
   // Visit each instruction in the function.
   for (MachineBasicBlock &MBB : MF) {
@@ -229,8 +233,8 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
       if (MI.isDebugInstr() || MI.isLabel())
         continue;
 
-      // Replace tee instructions with tee_local. The difference is that tee
-      // instructins have two defs, while tee_local instructions have one def
+      // Replace tee instructions with local.tee. The difference is that tee
+      // instructions have two defs, while local.tee instructions have one def
       // and an index of a local to write to.
       if (WebAssembly::isTee(MI)) {
         assert(MFI.isVRegStackified(MI.getOperand(0).getReg()));
@@ -249,7 +253,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
           MFI.stackifyVReg(NewReg);
         }
 
-        // Replace the TEE with a TEE_LOCAL.
+        // Replace the TEE with a LOCAL_TEE.
         unsigned LocalId =
             getLocalId(Reg2Local, CurLocal, MI.getOperand(1).getReg());
         unsigned Opc = getTeeLocalOpcode(RC);
@@ -263,7 +267,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
         continue;
       }
 
-      // Insert set_locals for any defs that aren't stackified yet. Currently
+      // Insert local.sets for any defs that aren't stackified yet. Currently
       // we handle at most one def.
       assert(MI.getDesc().getNumDefs() <= 1);
       if (MI.getDesc().getNumDefs() == 1) {
@@ -292,15 +296,16 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
                 .addReg(NewReg);
           }
           MI.getOperand(0).setReg(NewReg);
-          // This register operand is now being used by the inserted drop
-          // instruction, so make it undead.
+          // This register operand of the original instruction is now being used
+          // by the inserted drop or local.set instruction, so make it not dead
+          // yet.
           MI.getOperand(0).setIsDead(false);
           MFI.stackifyVReg(NewReg);
           Changed = true;
         }
       }
 
-      // Insert get_locals for any uses that aren't stackified yet.
+      // Insert local.gets for any uses that aren't stackified yet.
       MachineInstr *InsertPt = &MI;
       for (MachineOperand &MO : reverse(MI.explicit_uses())) {
         if (!MO.isReg())
@@ -314,15 +319,17 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
         if (MO.isDef()) {
           assert(MI.getOpcode() == TargetOpcode::INLINEASM);
           unsigned LocalId = getLocalId(Reg2Local, CurLocal, OldReg);
-          MRI.removeRegOperandFromUseList(&MO);
-          MO = MachineOperand::CreateImm(LocalId);
+          // If this register operand is tied to another operand, we can't
+          // change it to an immediate. Untie it first.
+          MI.untieRegOperand(MI.getOperandNo(&MO));
+          MO.ChangeToImmediate(LocalId);
           continue;
         }
 
         // If we see a stackified register, prepare to insert subsequent
-        // get_locals before the start of its tree.
+        // local.gets before the start of its tree.
         if (MFI.isVRegStackified(OldReg)) {
-          InsertPt = FindStartOfTree(MO, MRI, MFI);
+          InsertPt = findStartOfTree(MO, MRI, MFI);
           continue;
         }
 
@@ -330,12 +337,13 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
         // indices as immediates.
         if (MI.getOpcode() == TargetOpcode::INLINEASM) {
           unsigned LocalId = getLocalId(Reg2Local, CurLocal, OldReg);
-          MRI.removeRegOperandFromUseList(&MO);
-          MO = MachineOperand::CreateImm(LocalId);
+          // Untie it first if this reg operand is tied to another operand.
+          MI.untieRegOperand(MI.getOperandNo(&MO));
+          MO.ChangeToImmediate(LocalId);
           continue;
         }
 
-        // Insert a get_local.
+        // Insert a local.get.
         unsigned LocalId = getLocalId(Reg2Local, CurLocal, OldReg);
         const TargetRegisterClass *RC = MRI.getRegClass(OldReg);
         unsigned NewReg = MRI.createVirtualRegister(RC);
@@ -361,13 +369,13 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
   // Define the locals.
   // TODO: Sort the locals for better compression.
   MFI.setNumLocals(CurLocal - MFI.getParams().size());
-  for (size_t i = 0, e = MRI.getNumVirtRegs(); i < e; ++i) {
-    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
-    auto I = Reg2Local.find(Reg);
-    if (I == Reg2Local.end() || I->second < MFI.getParams().size())
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I < E; ++I) {
+    unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
+    auto RL = Reg2Local.find(Reg);
+    if (RL == Reg2Local.end() || RL->second < MFI.getParams().size())
       continue;
 
-    MFI.setLocal(I->second - MFI.getParams().size(),
+    MFI.setLocal(RL->second - MFI.getParams().size(),
                  typeForRegClass(MRI.getRegClass(Reg)));
     Changed = true;
   }
diff --git a/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index 566ef68c027d..3856700cca94 100644
--- a/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -37,7 +37,10 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/IR/PatternMatch.h"
+
 using namespace llvm;
+using namespace PatternMatch;
 
 #define DEBUG_TYPE "wasm-fastisel"
 
@@ -114,8 +117,8 @@ private:
   // Utility helper routines
   MVT::SimpleValueType getSimpleType(Type *Ty) {
     EVT VT = TLI.getValueType(DL, Ty, /*HandleUnknown=*/true);
-    return VT.isSimple() ? VT.getSimpleVT().SimpleTy :
-                           MVT::INVALID_SIMPLE_VALUE_TYPE;
+    return VT.isSimple() ? VT.getSimpleVT().SimpleTy
+                         : MVT::INVALID_SIMPLE_VALUE_TYPE;
   }
   MVT::SimpleValueType getLegalType(MVT::SimpleValueType VT) {
     switch (VT) {
@@ -138,6 +141,11 @@ private:
       if (Subtarget->hasSIMD128())
         return VT;
       break;
+    case MVT::v2i64:
+    case MVT::v2f64:
+      if (Subtarget->hasUnimplementedSIMD128())
+        return VT;
+      break;
     default:
       break;
     }
@@ -153,11 +161,9 @@ private:
                            MVT::SimpleValueType From);
   unsigned signExtendToI32(unsigned Reg, const Value *V,
                            MVT::SimpleValueType From);
-  unsigned zeroExtend(unsigned Reg, const Value *V,
-                      MVT::SimpleValueType From,
+  unsigned zeroExtend(unsigned Reg, const Value *V, MVT::SimpleValueType From,
                       MVT::SimpleValueType To);
-  unsigned signExtend(unsigned Reg, const Value *V,
-                      MVT::SimpleValueType From,
+  unsigned signExtend(unsigned Reg, const Value *V, MVT::SimpleValueType From,
                       MVT::SimpleValueType To);
   unsigned getRegForUnsignedValue(const Value *V);
   unsigned getRegForSignedValue(const Value *V);
@@ -374,14 +380,12 @@ void WebAssemblyFastISel::materializeLoadStoreOperands(Address &Addr) {
   if (Addr.isRegBase()) {
     unsigned Reg = Addr.getReg();
     if (Reg == 0) {
-      Reg = createResultReg(Subtarget->hasAddr64() ?
-                            &WebAssembly::I64RegClass :
-                            &WebAssembly::I32RegClass);
-      unsigned Opc = Subtarget->hasAddr64() ?
-                     WebAssembly::CONST_I64 :
-                     WebAssembly::CONST_I32;
+      Reg = createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
+                                                   : &WebAssembly::I32RegClass);
+      unsigned Opc = Subtarget->hasAddr64() ? WebAssembly::CONST_I64
+                                            : WebAssembly::CONST_I32;
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), Reg)
-         .addImm(0);
+          .addImm(0);
       Addr.setReg(Reg);
     }
   }
@@ -419,9 +423,10 @@ unsigned WebAssemblyFastISel::getRegForI1Value(const Value *V, bool &Not) {
         return getRegForValue(ICmp->getOperand(0));
       }
 
-  if (BinaryOperator::isNot(V) && V->getType()->isIntegerTy(32)) {
+  Value *NotV;
+  if (match(V, m_Not(m_Value(NotV))) && V->getType()->isIntegerTy(32)) {
     Not = true;
-    return getRegForValue(BinaryOperator::getNotArgument(V));
+    return getRegForValue(NotV);
   }
 
   Not = false;
@@ -438,13 +443,12 @@ unsigned WebAssemblyFastISel::zeroExtendToI32(unsigned Reg, const Value *V,
 
   switch (From) {
   case MVT::i1:
-    // If the value is naturally an i1, we don't need to mask it.
-    // TODO: Recursively examine selects, phis, and, or, xor, constants.
-    if (From == MVT::i1 && V != nullptr) {
-      if (isa<CmpInst>(V) ||
-          (isa<Argument>(V) && cast<Argument>(V)->hasZExtAttr()))
-        return copyValue(Reg);
-    }
+    // If the value is naturally an i1, we don't need to mask it. We only know
+    // if a value is naturally an i1 if it is definitely lowered by FastISel,
+    // not a DAG ISel fallback.
+    if (V != nullptr && isa<Argument>(V) && cast<Argument>(V)->hasZExtAttr())
+      return copyValue(Reg);
+    break;
   case MVT::i8:
   case MVT::i16:
     break;
@@ -457,13 +461,13 @@ unsigned WebAssemblyFastISel::zeroExtendToI32(unsigned Reg, const Value *V,
   unsigned Imm = createResultReg(&WebAssembly::I32RegClass);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(WebAssembly::CONST_I32), Imm)
-    .addImm(~(~uint64_t(0) << MVT(From).getSizeInBits()));
+      .addImm(~(~uint64_t(0) << MVT(From).getSizeInBits()));
 
   unsigned Result = createResultReg(&WebAssembly::I32RegClass);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(WebAssembly::AND_I32), Result)
-    .addReg(Reg)
-    .addReg(Imm);
+      .addReg(Reg)
+      .addReg(Imm);
 
   return Result;
 }
@@ -487,19 +491,19 @@ unsigned WebAssemblyFastISel::signExtendToI32(unsigned Reg, const Value *V,
   unsigned Imm = createResultReg(&WebAssembly::I32RegClass);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(WebAssembly::CONST_I32), Imm)
-    .addImm(32 - MVT(From).getSizeInBits());
+      .addImm(32 - MVT(From).getSizeInBits());
 
   unsigned Left = createResultReg(&WebAssembly::I32RegClass);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(WebAssembly::SHL_I32), Left)
-    .addReg(Reg)
-    .addReg(Imm);
+      .addReg(Reg)
+      .addReg(Imm);
 
   unsigned Right = createResultReg(&WebAssembly::I32RegClass);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(WebAssembly::SHR_S_I32), Right)
-    .addReg(Left)
-    .addReg(Imm);
+      .addReg(Left)
+      .addReg(Imm);
 
   return Right;
 }
@@ -562,8 +566,7 @@ unsigned WebAssemblyFastISel::getRegForSignedValue(const Value *V) {
 
 unsigned WebAssemblyFastISel::getRegForPromotedValue(const Value *V,
                                                      bool IsSigned) {
-  return IsSigned ? getRegForSignedValue(V) :
-                    getRegForUnsignedValue(V);
+  return IsSigned ? getRegForSignedValue(V) : getRegForUnsignedValue(V);
 }
 
 unsigned WebAssemblyFastISel::notValue(unsigned Reg) {
@@ -572,15 +575,15 @@ unsigned WebAssemblyFastISel::notValue(unsigned Reg) {
   unsigned NotReg = createResultReg(&WebAssembly::I32RegClass);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(WebAssembly::EQZ_I32), NotReg)
-    .addReg(Reg);
+      .addReg(Reg);
   return NotReg;
 }
 
 unsigned WebAssemblyFastISel::copyValue(unsigned Reg) {
   unsigned ResultReg = createResultReg(MRI.getRegClass(Reg));
-  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-          TII.get(WebAssembly::COPY), ResultReg)
-    .addReg(Reg);
+  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(WebAssembly::COPY),
+          ResultReg)
+      .addReg(Reg);
   return ResultReg;
 }
 
@@ -589,12 +592,11 @@ unsigned WebAssemblyFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
       FuncInfo.StaticAllocaMap.find(AI);
 
   if (SI != FuncInfo.StaticAllocaMap.end()) {
-    unsigned ResultReg = createResultReg(Subtarget->hasAddr64() ?
-                                         &WebAssembly::I64RegClass :
-                                         &WebAssembly::I32RegClass);
-    unsigned Opc = Subtarget->hasAddr64() ?
-                   WebAssembly::COPY_I64 :
-                   WebAssembly::COPY_I32;
+    unsigned ResultReg =
+        createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
+                                               : &WebAssembly::I32RegClass);
+    unsigned Opc =
+        Subtarget->hasAddr64() ? WebAssembly::COPY_I64 : WebAssembly::COPY_I32;
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
         .addFrameIndex(SI->second);
     return ResultReg;
@@ -605,14 +607,13 @@ unsigned WebAssemblyFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
 
 unsigned WebAssemblyFastISel::fastMaterializeConstant(const Constant *C) {
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) {
-    unsigned ResultReg = createResultReg(Subtarget->hasAddr64() ?
-                                         &WebAssembly::I64RegClass :
-                                         &WebAssembly::I32RegClass);
-    unsigned Opc = Subtarget->hasAddr64() ?
-                   WebAssembly::CONST_I64 :
-                   WebAssembly::CONST_I32;
+    unsigned ResultReg =
+        createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
+                                               : &WebAssembly::I32RegClass);
+    unsigned Opc = Subtarget->hasAddr64() ? WebAssembly::CONST_I64
+                                          : WebAssembly::CONST_I32;
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
-       .addGlobalAddress(GV);
+        .addGlobalAddress(GV);
     return ResultReg;
   }
 
@@ -651,19 +652,19 @@ bool WebAssemblyFastISel::fastLowerArguments() {
     case MVT::i8:
     case MVT::i16:
     case MVT::i32:
-      Opc = WebAssembly::ARGUMENT_I32;
+      Opc = WebAssembly::ARGUMENT_i32;
       RC = &WebAssembly::I32RegClass;
       break;
     case MVT::i64:
-      Opc = WebAssembly::ARGUMENT_I64;
+      Opc = WebAssembly::ARGUMENT_i64;
       RC = &WebAssembly::I64RegClass;
       break;
     case MVT::f32:
-      Opc = WebAssembly::ARGUMENT_F32;
+      Opc = WebAssembly::ARGUMENT_f32;
       RC = &WebAssembly::F32RegClass;
       break;
     case MVT::f64:
-      Opc = WebAssembly::ARGUMENT_F64;
+      Opc = WebAssembly::ARGUMENT_f64;
       RC = &WebAssembly::F64RegClass;
       break;
     case MVT::v16i8:
@@ -678,12 +679,20 @@ bool WebAssemblyFastISel::fastLowerArguments() {
       Opc = WebAssembly::ARGUMENT_v4i32;
       RC = &WebAssembly::V128RegClass;
       break;
+    case MVT::v2i64:
+      Opc = WebAssembly::ARGUMENT_v2i64;
+      RC = &WebAssembly::V128RegClass;
+      break;
     case MVT::v4f32:
       Opc = WebAssembly::ARGUMENT_v4f32;
       RC = &WebAssembly::V128RegClass;
       break;
+    case MVT::v2f64:
+      Opc = WebAssembly::ARGUMENT_v2f64;
+      RC = &WebAssembly::V128RegClass;
+      break;
     case MVT::ExceptRef:
-      Opc = WebAssembly::ARGUMENT_EXCEPT_REF;
+      Opc = WebAssembly::ARGUMENT_ExceptRef;
       RC = &WebAssembly::EXCEPT_REFRegClass;
       break;
     default:
@@ -691,7 +700,7 @@ bool WebAssemblyFastISel::fastLowerArguments() {
     }
     unsigned ResultReg = createResultReg(RC);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
-      .addImm(i);
+        .addImm(i);
     updateValueMap(&Arg, ResultReg);
 
     ++i;
@@ -710,7 +719,8 @@ bool WebAssemblyFastISel::fastLowerArguments() {
   }
 
   if (!F->getReturnType()->isVoidTy()) {
-    MVT::SimpleValueType RetTy = getLegalType(getSimpleType(F->getReturnType()));
+    MVT::SimpleValueType RetTy =
+        getLegalType(getSimpleType(F->getReturnType()));
     if (RetTy == MVT::INVALID_SIMPLE_VALUE_TYPE) {
       MFI->clearParamsAndResults();
       return false;
@@ -768,23 +778,33 @@ bool WebAssemblyFastISel::selectCall(const Instruction *I) {
       ResultReg = createResultReg(&WebAssembly::F64RegClass);
       break;
     case MVT::v16i8:
-      Opc =
-          IsDirect ? WebAssembly::CALL_v16i8 : WebAssembly::PCALL_INDIRECT_v16i8;
+      Opc = IsDirect ? WebAssembly::CALL_v16i8
+                     : WebAssembly::PCALL_INDIRECT_v16i8;
       ResultReg = createResultReg(&WebAssembly::V128RegClass);
       break;
     case MVT::v8i16:
-      Opc =
-          IsDirect ? WebAssembly::CALL_v8i16 : WebAssembly::PCALL_INDIRECT_v8i16;
+      Opc = IsDirect ? WebAssembly::CALL_v8i16
+                     : WebAssembly::PCALL_INDIRECT_v8i16;
       ResultReg = createResultReg(&WebAssembly::V128RegClass);
       break;
     case MVT::v4i32:
-      Opc =
-          IsDirect ? WebAssembly::CALL_v4i32 : WebAssembly::PCALL_INDIRECT_v4i32;
+      Opc = IsDirect ? WebAssembly::CALL_v4i32
+                     : WebAssembly::PCALL_INDIRECT_v4i32;
+      ResultReg = createResultReg(&WebAssembly::V128RegClass);
+      break;
+    case MVT::v2i64:
+      Opc = IsDirect ? WebAssembly::CALL_v2i64
+                     : WebAssembly::PCALL_INDIRECT_v2i64;
       ResultReg = createResultReg(&WebAssembly::V128RegClass);
       break;
     case MVT::v4f32:
-      Opc =
-          IsDirect ? WebAssembly::CALL_v4f32 : WebAssembly::PCALL_INDIRECT_v4f32;
+      Opc = IsDirect ? WebAssembly::CALL_v4f32
+                     : WebAssembly::PCALL_INDIRECT_v4f32;
+      ResultReg = createResultReg(&WebAssembly::V128RegClass);
+      break;
+    case MVT::v2f64:
+      Opc = IsDirect ? WebAssembly::CALL_v2f64
+                     : WebAssembly::PCALL_INDIRECT_v2f64;
       ResultReg = createResultReg(&WebAssembly::V128RegClass);
       break;
     case MVT::ExceptRef:
@@ -853,11 +873,11 @@ bool WebAssemblyFastISel::selectSelect(const Instruction *I) {
   const SelectInst *Select = cast<SelectInst>(I);
 
   bool Not;
-  unsigned CondReg  = getRegForI1Value(Select->getCondition(), Not);
+  unsigned CondReg = getRegForI1Value(Select->getCondition(), Not);
   if (CondReg == 0)
     return false;
 
-  unsigned TrueReg  = getRegForValue(Select->getTrueValue());
+  unsigned TrueReg = getRegForValue(Select->getTrueValue());
   if (TrueReg == 0)
     return false;
 
@@ -900,9 +920,9 @@ bool WebAssemblyFastISel::selectSelect(const Instruction *I) {
 
   unsigned ResultReg = createResultReg(RC);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
-    .addReg(TrueReg)
-    .addReg(FalseReg)
-    .addReg(CondReg);
+      .addReg(TrueReg)
+      .addReg(FalseReg)
+      .addReg(CondReg);
 
   updateValueMap(Select, ResultReg);
   return true;
@@ -1002,7 +1022,8 @@ bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
     Opc = I32 ? WebAssembly::LE_S_I32 : WebAssembly::LE_S_I64;
     isSigned = true;
     break;
-  default: return false;
+  default:
+    return false;
   }
 
   unsigned LHS = getRegForPromotedValue(ICmp->getOperand(0), isSigned);
@@ -1210,7 +1231,8 @@ bool WebAssemblyFastISel::selectStore(const Instruction *I) {
   case MVT::f64:
     Opc = WebAssembly::STORE_F64;
     break;
-  default: return false;
+  default:
+    return false;
   }
 
   materializeLoadStoreOperands(Addr);
@@ -1275,8 +1297,10 @@ bool WebAssemblyFastISel::selectRet(const Instruction *I) {
 
   unsigned Opc;
   switch (getSimpleType(RV->getType())) {
-  case MVT::i1: case MVT::i8:
-  case MVT::i16: case MVT::i32:
+  case MVT::i1:
+  case MVT::i8:
+  case MVT::i16:
+  case MVT::i32:
     Opc = WebAssembly::RETURN_I32;
     break;
   case MVT::i64:
@@ -1297,13 +1321,20 @@ bool WebAssemblyFastISel::selectRet(const Instruction *I) {
   case MVT::v4i32:
     Opc = WebAssembly::RETURN_v4i32;
     break;
+  case MVT::v2i64:
+    Opc = WebAssembly::RETURN_v2i64;
+    break;
   case MVT::v4f32:
     Opc = WebAssembly::RETURN_v4f32;
     break;
+  case MVT::v2f64:
+    Opc = WebAssembly::RETURN_v2f64;
+    break;
   case MVT::ExceptRef:
     Opc = WebAssembly::RETURN_EXCEPT_REF;
     break;
-  default: return false;
+  default:
+    return false;
   }
 
   unsigned Reg;
@@ -1333,19 +1364,32 @@ bool WebAssemblyFastISel::fastSelectInstruction(const Instruction *I) {
     if (selectCall(I))
       return true;
     break;
-  case Instruction::Select:      return selectSelect(I);
-  case Instruction::Trunc:       return selectTrunc(I);
-  case Instruction::ZExt:        return selectZExt(I);
-  case Instruction::SExt:        return selectSExt(I);
-  case Instruction::ICmp:        return selectICmp(I);
-  case Instruction::FCmp:        return selectFCmp(I);
-  case Instruction::BitCast:     return selectBitCast(I);
-  case Instruction::Load:        return selectLoad(I);
-  case Instruction::Store:       return selectStore(I);
-  case Instruction::Br:          return selectBr(I);
-  case Instruction::Ret:         return selectRet(I);
-  case Instruction::Unreachable: return selectUnreachable(I);
-  default: break;
+  case Instruction::Select:
+    return selectSelect(I);
+  case Instruction::Trunc:
+    return selectTrunc(I);
+  case Instruction::ZExt:
+    return selectZExt(I);
+  case Instruction::SExt:
+    return selectSExt(I);
+  case Instruction::ICmp:
+    return selectICmp(I);
+  case Instruction::FCmp:
+    return selectFCmp(I);
+  case Instruction::BitCast:
+    return selectBitCast(I);
+  case Instruction::Load:
+    return selectLoad(I);
+  case Instruction::Store:
+    return selectStore(I);
+  case Instruction::Br:
+    return selectBr(I);
+  case Instruction::Ret:
+    return selectRet(I);
+  case Instruction::Unreachable:
+    return selectUnreachable(I);
+  default:
+    break;
   }
 
   // Fall back to target-independent instruction selection.
diff --git a/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp b/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
index d5e47ee82513..1a416520f97d 100644
--- a/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
@@ -36,10 +36,10 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-fix-function-bitcasts"
 
-static cl::opt<bool> TemporaryWorkarounds(
-  "wasm-temporary-workarounds",
-  cl::desc("Apply certain temporary workarounds"),
-  cl::init(true), cl::Hidden);
+static cl::opt<bool>
+    TemporaryWorkarounds("wasm-temporary-workarounds",
+                         cl::desc("Apply certain temporary workarounds"),
+                         cl::init(true), cl::Hidden);
 
 namespace {
 class FixFunctionBitcasts final : public ModulePass {
@@ -103,14 +103,29 @@ static void FindUses(Value *V, Function &F,
 //  - Return value is not needed: drop it
 //  - Return value needed but not present: supply an undef
 //
-// For now, return nullptr without creating a wrapper if the wrapper cannot
-// be generated due to incompatible types.
+// If the all the argument types of trivially castable to one another (i.e.
+// I32 vs pointer type) then we don't create a wrapper at all (return nullptr
+// instead).
+//
+// If there is a type mismatch that we know would result in an invalid wasm
+// module then generate wrapper that contains unreachable (i.e. abort at
+// runtime).  Such programs are deep into undefined behaviour territory,
+// but we choose to fail at runtime rather than generate and invalid module
+// or fail at compiler time.  The reason we delay the error is that we want
+// to support the CMake which expects to be able to compile and link programs
+// that refer to functions with entirely incorrect signatures (this is how
+// CMake detects the existence of a function in a toolchain).
+//
+// For bitcasts that involve struct types we don't know at this stage if they
+// would be equivalent at the wasm level and so we can't know if we need to
+// generate a wrapper.
 static Function *CreateWrapper(Function *F, FunctionType *Ty) {
   Module *M = F->getParent();
 
-  Function *Wrapper =
-      Function::Create(Ty, Function::PrivateLinkage, "bitcast", M);
+  Function *Wrapper = Function::Create(Ty, Function::PrivateLinkage,
+                                       F->getName() + "_bitcast", M);
   BasicBlock *BB = BasicBlock::Create(M->getContext(), "body", Wrapper);
+  const DataLayout &DL = BB->getModule()->getDataLayout();
 
   // Determine what arguments to pass.
   SmallVector<Value *, 4> Args;
@@ -118,38 +133,103 @@ static Function *CreateWrapper(Function *F, FunctionType *Ty) {
   Function::arg_iterator AE = Wrapper->arg_end();
   FunctionType::param_iterator PI = F->getFunctionType()->param_begin();
   FunctionType::param_iterator PE = F->getFunctionType()->param_end();
+  bool TypeMismatch = false;
+  bool WrapperNeeded = false;
+
+  Type *ExpectedRtnType = F->getFunctionType()->getReturnType();
+  Type *RtnType = Ty->getReturnType();
+
+  if ((F->getFunctionType()->getNumParams() != Ty->getNumParams()) ||
+      (F->getFunctionType()->isVarArg() != Ty->isVarArg()) ||
+      (ExpectedRtnType != RtnType))
+    WrapperNeeded = true;
+
   for (; AI != AE && PI != PE; ++AI, ++PI) {
-    if (AI->getType() != *PI) {
-      Wrapper->eraseFromParent();
-      return nullptr;
+    Type *ArgType = AI->getType();
+    Type *ParamType = *PI;
+
+    if (ArgType == ParamType) {
+      Args.push_back(&*AI);
+    } else {
+      if (CastInst::isBitOrNoopPointerCastable(ArgType, ParamType, DL)) {
+        Instruction *PtrCast =
+            CastInst::CreateBitOrPointerCast(AI, ParamType, "cast");
+        BB->getInstList().push_back(PtrCast);
+        Args.push_back(PtrCast);
+      } else if (ArgType->isStructTy() || ParamType->isStructTy()) {
+        LLVM_DEBUG(dbgs() << "CreateWrapper: struct param type in bitcast: "
+                          << F->getName() << "\n");
+        WrapperNeeded = false;
+      } else {
+        LLVM_DEBUG(dbgs() << "CreateWrapper: arg type mismatch calling: "
+                          << F->getName() << "\n");
+        LLVM_DEBUG(dbgs() << "Arg[" << Args.size() << "] Expected: "
+                          << *ParamType << " Got: " << *ArgType << "\n");
+        TypeMismatch = true;
+        break;
+      }
     }
-    Args.push_back(&*AI);
   }
-  for (; PI != PE; ++PI)
-    Args.push_back(UndefValue::get(*PI));
-  if (F->isVarArg())
-    for (; AI != AE; ++AI)
-      Args.push_back(&*AI);
 
-  CallInst *Call = CallInst::Create(F, Args, "", BB);
+  if (WrapperNeeded && !TypeMismatch) {
+    for (; PI != PE; ++PI)
+      Args.push_back(UndefValue::get(*PI));
+    if (F->isVarArg())
+      for (; AI != AE; ++AI)
+        Args.push_back(&*AI);
 
-  // Determine what value to return.
-  if (Ty->getReturnType()->isVoidTy())
-    ReturnInst::Create(M->getContext(), BB);
-  else if (F->getFunctionType()->getReturnType()->isVoidTy())
-    ReturnInst::Create(M->getContext(), UndefValue::get(Ty->getReturnType()),
-                       BB);
-  else if (F->getFunctionType()->getReturnType() == Ty->getReturnType())
-    ReturnInst::Create(M->getContext(), Call, BB);
-  else {
+    CallInst *Call = CallInst::Create(F, Args, "", BB);
+
+    Type *ExpectedRtnType = F->getFunctionType()->getReturnType();
+    Type *RtnType = Ty->getReturnType();
+    // Determine what value to return.
+    if (RtnType->isVoidTy()) {
+      ReturnInst::Create(M->getContext(), BB);
+    } else if (ExpectedRtnType->isVoidTy()) {
+      LLVM_DEBUG(dbgs() << "Creating dummy return: " << *RtnType << "\n");
+      ReturnInst::Create(M->getContext(), UndefValue::get(RtnType), BB);
+    } else if (RtnType == ExpectedRtnType) {
+      ReturnInst::Create(M->getContext(), Call, BB);
+    } else if (CastInst::isBitOrNoopPointerCastable(ExpectedRtnType, RtnType,
+                                                    DL)) {
+      Instruction *Cast =
+          CastInst::CreateBitOrPointerCast(Call, RtnType, "cast");
+      BB->getInstList().push_back(Cast);
+      ReturnInst::Create(M->getContext(), Cast, BB);
+    } else if (RtnType->isStructTy() || ExpectedRtnType->isStructTy()) {
+      LLVM_DEBUG(dbgs() << "CreateWrapper: struct return type in bitcast: "
+                        << F->getName() << "\n");
+      WrapperNeeded = false;
+    } else {
+      LLVM_DEBUG(dbgs() << "CreateWrapper: return type mismatch calling: "
+                        << F->getName() << "\n");
+      LLVM_DEBUG(dbgs() << "Expected: " << *ExpectedRtnType
+                        << " Got: " << *RtnType << "\n");
+      TypeMismatch = true;
+    }
+  }
+
+  if (TypeMismatch) {
+    // Create a new wrapper that simply contains `unreachable`.
+    Wrapper->eraseFromParent();
+    Wrapper = Function::Create(Ty, Function::PrivateLinkage,
+                               F->getName() + "_bitcast_invalid", M);
+    BasicBlock *BB = BasicBlock::Create(M->getContext(), "body", Wrapper);
+    new UnreachableInst(M->getContext(), BB);
+    Wrapper->setName(F->getName() + "_bitcast_invalid");
+  } else if (!WrapperNeeded) {
+    LLVM_DEBUG(dbgs() << "CreateWrapper: no wrapper needed: " << F->getName()
+                      << "\n");
     Wrapper->eraseFromParent();
     return nullptr;
   }
-
+  LLVM_DEBUG(dbgs() << "CreateWrapper: " << F->getName() << "\n");
   return Wrapper;
 }
 
 bool FixFunctionBitcasts::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "********** Fix Function Bitcasts **********\n");
+
   Function *Main = nullptr;
   CallInst *CallMain = nullptr;
   SmallVector<std::pair<Use *, Function *>, 0> Uses;
@@ -166,19 +246,17 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
     if (!TemporaryWorkarounds && !F.isDeclaration() && F.getName() == "main") {
       Main = &F;
       LLVMContext &C = M.getContext();
-      Type *MainArgTys[] = {
-        PointerType::get(Type::getInt8PtrTy(C), 0),
-        Type::getInt32Ty(C)
-      };
+      Type *MainArgTys[] = {Type::getInt32Ty(C),
+                            PointerType::get(Type::getInt8PtrTy(C), 0)};
       FunctionType *MainTy = FunctionType::get(Type::getInt32Ty(C), MainArgTys,
                                                /*isVarArg=*/false);
       if (F.getFunctionType() != MainTy) {
-        Value *Args[] = {
-          UndefValue::get(MainArgTys[0]),
-          UndefValue::get(MainArgTys[1])
-        };
-        Value *Casted = ConstantExpr::getBitCast(Main,
-                                                 PointerType::get(MainTy, 0));
+        LLVM_DEBUG(dbgs() << "Found `main` function with incorrect type: "
+                          << *F.getFunctionType() << "\n");
+        Value *Args[] = {UndefValue::get(MainArgTys[0]),
+                         UndefValue::get(MainArgTys[1])};
+        Value *Casted =
+            ConstantExpr::getBitCast(Main, PointerType::get(MainTy, 0));
         CallMain = CallInst::Create(Casted, Args, "call_main");
         Use *UseMain = &CallMain->getOperandUse(2);
         Uses.push_back(std::make_pair(UseMain, &F));
@@ -200,11 +278,6 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
     if (!Ty)
       continue;
 
-    // Bitcasted vararg functions occur in Emscripten's implementation of
-    // EM_ASM, so suppress wrappers for them for now.
-    if (TemporaryWorkarounds && (Ty->isVarArg() || F->isVarArg()))
-      continue;
-
     auto Pair = Wrappers.insert(std::make_pair(std::make_pair(F, Ty), nullptr));
     if (Pair.second)
       Pair.first->second = CreateWrapper(F, Ty);
diff --git a/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp b/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
index bea027be7711..108f2879a071 100644
--- a/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
@@ -8,8 +8,8 @@
 //===----------------------------------------------------------------------===//
 ///
 /// \file
-/// This file implements a pass that transforms irreducible control flow
-/// into reducible control flow. Irreducible control flow means multiple-entry
+/// This file implements a pass that transforms irreducible control flow into
+/// reducible control flow. Irreducible control flow means multiple-entry
 /// loops; they appear as CFG cycles that are not recorded in MachineLoopInfo
 /// due to being unnatural.
 ///
@@ -17,12 +17,36 @@
 /// it linearizes control flow, turning diamonds into two triangles, which is
 /// both unnecessary and undesirable for WebAssembly.
 ///
-/// TODO: The transformation implemented here handles all irreducible control
-/// flow, without exponential code-size expansion, though it does so by creating
-/// inefficient code in many cases. Ideally, we should add other
-/// transformations, including code-duplicating cases, which can be more
-/// efficient in common cases, and they can fall back to this conservative
-/// implementation as needed.
+/// The big picture: Ignoring natural loops (seeing them monolithically), we
+/// find all the blocks which can return to themselves ("loopers"). Loopers
+/// reachable from the non-loopers are loop entries: if there are 2 or more,
+/// then we have irreducible control flow. We fix that as follows: a new block
+/// is created that can dispatch to each of the loop entries, based on the
+/// value of a label "helper" variable, and we replace direct branches to the
+/// entries with assignments to the label variable and a branch to the dispatch
+/// block. Then the dispatch block is the single entry in a new natural loop.
+///
+/// This is similar to what the Relooper [1] does, both identify looping code
+/// that requires multiple entries, and resolve it in a similar way. In
+/// Relooper terminology, we implement a Multiple shape in a Loop shape. Note
+/// also that like the Relooper, we implement a "minimal" intervention: we only
+/// use the "label" helper for the blocks we absolutely must and no others. We
+/// also prioritize code size and do not perform node splitting (i.e. we don't
+/// duplicate code in order to resolve irreducibility).
+///
+/// The difference between this code and the Relooper is that the Relooper also
+/// generates ifs and loops and works in a recursive manner, knowing at each
+/// point what the entries are, and recursively breaks down the problem. Here
+/// we just want to resolve irreducible control flow, and we also want to use
+/// as much LLVM infrastructure as possible. So we use the MachineLoopInfo to
+/// identify natural loops, etc., and we start with the whole CFG and must
+/// identify both the looping code and its entries.
+///
+/// [1] Alon Zakai. 2011. Emscripten: an LLVM-to-JavaScript compiler. In
+/// Proceedings of the ACM international conference companion on Object oriented
+/// programming systems languages and applications companion (SPLASH '11). ACM,
+/// New York, NY, USA, 301-312. DOI=10.1145/2048147.2048224
+/// http://doi.acm.org/10.1145/2048147.2048224
 ///
 //===----------------------------------------------------------------------===//
 
@@ -46,141 +70,203 @@ using namespace llvm;
 #define DEBUG_TYPE "wasm-fix-irreducible-control-flow"
 
 namespace {
-class WebAssemblyFixIrreducibleControlFlow final : public MachineFunctionPass {
-  StringRef getPassName() const override {
-    return "WebAssembly Fix Irreducible Control Flow";
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    AU.addRequired<MachineDominatorTree>();
-    AU.addPreserved<MachineDominatorTree>();
-    AU.addRequired<MachineLoopInfo>();
-    AU.addPreserved<MachineLoopInfo>();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  bool VisitLoop(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop);
 
+class LoopFixer {
 public:
-  static char ID; // Pass identification, replacement for typeid
-  WebAssemblyFixIrreducibleControlFlow() : MachineFunctionPass(ID) {}
-};
-} // end anonymous namespace
+  LoopFixer(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop)
+      : MF(MF), MLI(MLI), Loop(Loop) {}
 
-char WebAssemblyFixIrreducibleControlFlow::ID = 0;
-INITIALIZE_PASS(WebAssemblyFixIrreducibleControlFlow, DEBUG_TYPE,
-                "Removes irreducible control flow", false, false)
+  // Run the fixer on the given inputs. Returns whether changes were made.
+  bool run();
 
-FunctionPass *llvm::createWebAssemblyFixIrreducibleControlFlow() {
-  return new WebAssemblyFixIrreducibleControlFlow();
-}
+private:
+  MachineFunction &MF;
+  MachineLoopInfo &MLI;
+  MachineLoop *Loop;
 
-namespace {
+  MachineBasicBlock *Header;
+  SmallPtrSet<MachineBasicBlock *, 4> LoopBlocks;
 
-/// A utility for walking the blocks of a loop, handling a nested inner
-/// loop as a monolithic conceptual block.
-class MetaBlock {
-  MachineBasicBlock *Block;
-  SmallVector<MachineBasicBlock *, 2> Preds;
-  SmallVector<MachineBasicBlock *, 2> Succs;
+  using BlockSet = SmallPtrSet<MachineBasicBlock *, 4>;
+  DenseMap<MachineBasicBlock *, BlockSet> Reachable;
 
-public:
-  explicit MetaBlock(MachineBasicBlock *MBB)
-      : Block(MBB), Preds(MBB->pred_begin(), MBB->pred_end()),
-        Succs(MBB->succ_begin(), MBB->succ_end()) {}
+  // The worklist contains pairs of recent additions, (a, b), where we just
+  // added a link a => b.
+  using BlockPair = std::pair<MachineBasicBlock *, MachineBasicBlock *>;
+  SmallVector<BlockPair, 4> WorkList;
 
-  explicit MetaBlock(MachineLoop *Loop) : Block(Loop->getHeader()) {
-    Loop->getExitBlocks(Succs);
-    for (MachineBasicBlock *Pred : Block->predecessors())
-      if (!Loop->contains(Pred))
-        Preds.push_back(Pred);
+  // Get a canonical block to represent a block or a loop: the block, or if in
+  // an inner loop, the loop header, of it in an outer loop scope, we can
+  // ignore it. We need to call this on all blocks we work on.
+  MachineBasicBlock *canonicalize(MachineBasicBlock *MBB) {
+    MachineLoop *InnerLoop = MLI.getLoopFor(MBB);
+    if (InnerLoop == Loop) {
+      return MBB;
+    } else {
+      // This is either in an outer or an inner loop, and not in ours.
+      if (!LoopBlocks.count(MBB)) {
+        // It's in outer code, ignore it.
+        return nullptr;
+      }
+      assert(InnerLoop);
+      // It's in an inner loop, canonicalize it to the header of that loop.
+      return InnerLoop->getHeader();
+    }
   }
 
-  MachineBasicBlock *getBlock() const { return Block; }
-
-  const SmallVectorImpl<MachineBasicBlock *> &predecessors() const {
-    return Preds;
-  }
-  const SmallVectorImpl<MachineBasicBlock *> &successors() const {
-    return Succs;
+  // For a successor we can additionally ignore it if it's a branch back to a
+  // natural loop top, as when we are in the scope of a loop, we just care
+  // about internal irreducibility, and can ignore the loop we are in. We need
+  // to call this on all blocks in a context where they are a successor.
+  MachineBasicBlock *canonicalizeSuccessor(MachineBasicBlock *MBB) {
+    if (Loop && MBB == Loop->getHeader()) {
+      // Ignore branches going to the loop's natural header.
+      return nullptr;
+    }
+    return canonicalize(MBB);
   }
 
-  bool operator==(const MetaBlock &MBB) { return Block == MBB.Block; }
-  bool operator!=(const MetaBlock &MBB) { return Block != MBB.Block; }
+  // Potentially insert a new reachable edge, and if so, note it as further
+  // work.
+  void maybeInsert(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
+    assert(MBB == canonicalize(MBB));
+    assert(Succ);
+    // Succ may not be interesting as a sucessor.
+    Succ = canonicalizeSuccessor(Succ);
+    if (!Succ)
+      return;
+    if (Reachable[MBB].insert(Succ).second) {
+      // For there to be further work, it means that we have
+      //   X => MBB => Succ
+      // for some other X, and in that case X => Succ would be a new edge for
+      // us to discover later. However, if we don't care about MBB as a
+      // successor, then we don't care about that anyhow.
+      if (canonicalizeSuccessor(MBB)) {
+        WorkList.emplace_back(MBB, Succ);
+      }
+    }
+  }
 };
 
-class SuccessorList final : public MetaBlock {
-  size_t Index;
-  size_t Num;
+bool LoopFixer::run() {
+  Header = Loop ? Loop->getHeader() : &*MF.begin();
 
-public:
-  explicit SuccessorList(MachineBasicBlock *MBB)
-      : MetaBlock(MBB), Index(0), Num(successors().size()) {}
+  // Identify all the blocks in this loop scope.
+  if (Loop) {
+    for (auto *MBB : Loop->getBlocks()) {
+      LoopBlocks.insert(MBB);
+    }
+  } else {
+    for (auto &MBB : MF) {
+      LoopBlocks.insert(&MBB);
+    }
+  }
 
-  explicit SuccessorList(MachineLoop *Loop)
-      : MetaBlock(Loop), Index(0), Num(successors().size()) {}
+  // Compute which (canonicalized) blocks each block can reach.
 
-  bool HasNext() const { return Index != Num; }
+  // Add all the initial work.
+  for (auto *MBB : LoopBlocks) {
+    MachineLoop *InnerLoop = MLI.getLoopFor(MBB);
 
-  MachineBasicBlock *Next() {
-    assert(HasNext());
-    return successors()[Index++];
+    if (InnerLoop == Loop) {
+      for (auto *Succ : MBB->successors()) {
+        maybeInsert(MBB, Succ);
+      }
+    } else {
+      // It can't be in an outer loop - we loop on LoopBlocks - and so it must
+      // be an inner loop.
+      assert(InnerLoop);
+      // Check if we are the canonical block for this loop.
+      if (canonicalize(MBB) != MBB) {
+        continue;
+      }
+      // The successors are those of the loop.
+      SmallVector<MachineBasicBlock *, 2> ExitBlocks;
+      InnerLoop->getExitBlocks(ExitBlocks);
+      for (auto *Succ : ExitBlocks) {
+        maybeInsert(MBB, Succ);
+      }
+    }
   }
-};
 
-} // end anonymous namespace
+  // Do work until we are all done.
+  while (!WorkList.empty()) {
+    MachineBasicBlock *MBB;
+    MachineBasicBlock *Succ;
+    std::tie(MBB, Succ) = WorkList.pop_back_val();
+    // The worklist item is an edge we just added, so it must have valid blocks
+    // (and not something canonicalized to nullptr).
+    assert(MBB);
+    assert(Succ);
+    // The successor in that pair must also be a valid successor.
+    assert(MBB == canonicalizeSuccessor(MBB));
+    // We recently added MBB => Succ, and that means we may have enabled
+    // Pred => MBB => Succ. Check all the predecessors. Note that our loop here
+    // is correct for both a block and a block representing a loop, as the loop
+    // is natural and so the predecessors are all predecessors of the loop
+    // header, which is the block we have here.
+    for (auto *Pred : MBB->predecessors()) {
+      // Canonicalize, make sure it's relevant, and check it's not the same
+      // block (an update to the block itself doesn't help compute that same
+      // block).
+      Pred = canonicalize(Pred);
+      if (Pred && Pred != MBB) {
+        maybeInsert(Pred, Succ);
+      }
+    }
+  }
 
-bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
-                                                     MachineLoopInfo &MLI,
-                                                     MachineLoop *Loop) {
-  MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin();
-  SetVector<MachineBasicBlock *> RewriteSuccs;
+  // It's now trivial to identify the loopers.
+  SmallPtrSet<MachineBasicBlock *, 4> Loopers;
+  for (auto MBB : LoopBlocks) {
+    if (Reachable[MBB].count(MBB)) {
+      Loopers.insert(MBB);
+    }
+  }
+  // The header cannot be a looper. At the toplevel, LLVM does not allow the
+  // entry to be in a loop, and in a natural loop we should ignore the header.
+  assert(Loopers.count(Header) == 0);
 
-  // DFS through Loop's body, looking for irreducible control flow. Loop is
-  // natural, and we stay in its body, and we treat any nested loops
-  // monolithically, so any cycles we encounter indicate irreducibility.
-  SmallPtrSet<MachineBasicBlock *, 8> OnStack;
-  SmallPtrSet<MachineBasicBlock *, 8> Visited;
-  SmallVector<SuccessorList, 4> LoopWorklist;
-  LoopWorklist.push_back(SuccessorList(Header));
-  OnStack.insert(Header);
-  Visited.insert(Header);
-  while (!LoopWorklist.empty()) {
-    SuccessorList &Top = LoopWorklist.back();
-    if (Top.HasNext()) {
-      MachineBasicBlock *Next = Top.Next();
-      if (Next == Header || (Loop && !Loop->contains(Next)))
-        continue;
-      if (LLVM_LIKELY(OnStack.insert(Next).second)) {
-        if (!Visited.insert(Next).second) {
-          OnStack.erase(Next);
-          continue;
-        }
-        MachineLoop *InnerLoop = MLI.getLoopFor(Next);
-        if (InnerLoop != Loop)
-          LoopWorklist.push_back(SuccessorList(InnerLoop));
-        else
-          LoopWorklist.push_back(SuccessorList(Next));
-      } else {
-        RewriteSuccs.insert(Top.getBlock());
+  // Find the entries, loopers reachable from non-loopers.
+  SmallPtrSet<MachineBasicBlock *, 4> Entries;
+  SmallVector<MachineBasicBlock *, 4> SortedEntries;
+  for (auto *Looper : Loopers) {
+    for (auto *Pred : Looper->predecessors()) {
+      Pred = canonicalize(Pred);
+      if (Pred && !Loopers.count(Pred)) {
+        Entries.insert(Looper);
+        SortedEntries.push_back(Looper);
+        break;
       }
-      continue;
     }
-    OnStack.erase(Top.getBlock());
-    LoopWorklist.pop_back();
   }
 
-  // Most likely, we didn't find any irreducible control flow.
-  if (LLVM_LIKELY(RewriteSuccs.empty()))
+  // Check if we found irreducible control flow.
+  if (LLVM_LIKELY(Entries.size() <= 1))
     return false;
 
-  LLVM_DEBUG(dbgs() << "Irreducible control flow detected!\n");
+  // Sort the entries to ensure a deterministic build.
+  llvm::sort(SortedEntries,
+             [&](const MachineBasicBlock *A, const MachineBasicBlock *B) {
+               auto ANum = A->getNumber();
+               auto BNum = B->getNumber();
+               return ANum < BNum;
+             });
 
-  // Ok. We have irreducible control flow! Create a dispatch block which will
-  // contains a jump table to any block in the problematic set of blocks.
+#ifndef NDEBUG
+  for (auto Block : SortedEntries)
+    assert(Block->getNumber() != -1);
+  if (SortedEntries.size() > 1) {
+    for (auto I = SortedEntries.begin(), E = SortedEntries.end() - 1;
+         I != E; ++I) {
+      auto ANum = (*I)->getNumber();
+      auto BNum = (*(std::next(I)))->getNumber();
+      assert(ANum != BNum);
+    }
+  }
+#endif
+
+  // Create a dispatch block which will contain a jump table to the entries.
   MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock();
   MF.insert(MF.end(), Dispatch);
   MLI.changeLoopFor(Dispatch, Loop);
@@ -196,43 +282,43 @@ bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
   unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
   MIB.addReg(Reg);
 
-  // Collect all the blocks which need to have their successors rewritten,
-  // add the successors to the jump table, and remember their index.
+  // Compute the indices in the superheader, one for each bad block, and
+  // add them as successors.
   DenseMap<MachineBasicBlock *, unsigned> Indices;
-  SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(),
-                                                   RewriteSuccs.end());
-  while (!SuccWorklist.empty()) {
-    MachineBasicBlock *MBB = SuccWorklist.pop_back_val();
+  for (auto *MBB : SortedEntries) {
     auto Pair = Indices.insert(std::make_pair(MBB, 0));
-    if (!Pair.second)
+    if (!Pair.second) {
       continue;
+    }
 
     unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
-    LLVM_DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index
-                      << "\n");
-
     Pair.first->second = Index;
-    for (auto Pred : MBB->predecessors())
-      RewriteSuccs.insert(Pred);
 
     MIB.addMBB(MBB);
     Dispatch->addSuccessor(MBB);
+  }
 
-    MetaBlock Meta(MBB);
-    for (auto *Succ : Meta.successors())
-      if (Succ != Header && (!Loop || Loop->contains(Succ)))
-        SuccWorklist.push_back(Succ);
+  // Rewrite the problematic successors for every block that wants to reach the
+  // bad blocks. For simplicity, we just introduce a new block for every edge
+  // we need to rewrite. (Fancier things are possible.)
+
+  SmallVector<MachineBasicBlock *, 4> AllPreds;
+  for (auto *MBB : SortedEntries) {
+    for (auto *Pred : MBB->predecessors()) {
+      if (Pred != Dispatch) {
+        AllPreds.push_back(Pred);
+      }
+    }
   }
 
-  // Rewrite the problematic successors for every block in RewriteSuccs.
-  // For simplicity, we just introduce a new block for every edge we need to
-  // rewrite. Fancier things are possible.
-  for (MachineBasicBlock *MBB : RewriteSuccs) {
+  for (MachineBasicBlock *MBB : AllPreds) {
     DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map;
     for (auto *Succ : MBB->successors()) {
-      if (!Indices.count(Succ))
+      if (!Entries.count(Succ)) {
         continue;
+      }
 
+      // This is a successor we need to rewrite.
       MachineBasicBlock *Split = MF.CreateMachineBasicBlock();
       MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ)
                                              : MF.end(),
@@ -266,6 +352,55 @@ bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
   return true;
 }
 
+class WebAssemblyFixIrreducibleControlFlow final : public MachineFunctionPass {
+  StringRef getPassName() const override {
+    return "WebAssembly Fix Irreducible Control Flow";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<MachineDominatorTree>();
+    AU.addPreserved<MachineDominatorTree>();
+    AU.addRequired<MachineLoopInfo>();
+    AU.addPreserved<MachineLoopInfo>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  bool runIteration(MachineFunction &MF, MachineLoopInfo &MLI) {
+    // Visit the function body, which is identified as a null loop.
+    if (LoopFixer(MF, MLI, nullptr).run()) {
+      return true;
+    }
+
+    // Visit all the loops.
+    SmallVector<MachineLoop *, 8> Worklist(MLI.begin(), MLI.end());
+    while (!Worklist.empty()) {
+      MachineLoop *Loop = Worklist.pop_back_val();
+      Worklist.append(Loop->begin(), Loop->end());
+      if (LoopFixer(MF, MLI, Loop).run()) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  WebAssemblyFixIrreducibleControlFlow() : MachineFunctionPass(ID) {}
+};
+} // end anonymous namespace
+
+char WebAssemblyFixIrreducibleControlFlow::ID = 0;
+INITIALIZE_PASS(WebAssemblyFixIrreducibleControlFlow, DEBUG_TYPE,
+                "Removes irreducible control flow", false, false)
+
+FunctionPass *llvm::createWebAssemblyFixIrreducibleControlFlow() {
+  return new WebAssemblyFixIrreducibleControlFlow();
+}
+
 bool WebAssemblyFixIrreducibleControlFlow::runOnMachineFunction(
     MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "********** Fixing Irreducible Control Flow **********\n"
@@ -275,24 +410,19 @@ bool WebAssemblyFixIrreducibleControlFlow::runOnMachineFunction(
   bool Changed = false;
   auto &MLI = getAnalysis<MachineLoopInfo>();
 
-  // Visit the function body, which is identified as a null loop.
-  Changed |= VisitLoop(MF, MLI, nullptr);
-
-  // Visit all the loops.
-  SmallVector<MachineLoop *, 8> Worklist(MLI.begin(), MLI.end());
-  while (!Worklist.empty()) {
-    MachineLoop *CurLoop = Worklist.pop_back_val();
-    Worklist.append(CurLoop->begin(), CurLoop->end());
-    Changed |= VisitLoop(MF, MLI, CurLoop);
-  }
-
-  // If we made any changes, completely recompute everything.
-  if (LLVM_UNLIKELY(Changed)) {
-    LLVM_DEBUG(dbgs() << "Recomputing dominators and loops.\n");
+  // When we modify something, bail out and recompute MLI, then start again, as
+  // we create a new natural loop when we resolve irreducible control flow, and
+  // other loops may become nested in it, etc. In practice this is not an issue
+  // because irreducible control flow is rare, only very few cycles are needed
+  // here.
+  while (LLVM_UNLIKELY(runIteration(MF, MLI))) {
+    // We rewrote part of the function; recompute MLI and start again.
+    LLVM_DEBUG(dbgs() << "Recomputing loops.\n");
     MF.getRegInfo().invalidateLiveness();
     MF.RenumberBlocks();
     getAnalysis<MachineDominatorTree>().runOnMachineFunction(MF);
     MLI.runOnMachineFunction(MF);
+    Changed = true;
   }
 
   return Changed;
diff --git a/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp b/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
index 052c94e9d6a9..2d5aff28d27b 100644
--- a/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
@@ -30,6 +30,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/Debug.h"
 using namespace llvm;
 
@@ -42,8 +43,7 @@ using namespace llvm;
 /// require stricter alignment than the stack pointer itself.  Because we need
 /// to shift the stack pointer by some unknown amount to force the alignment,
 /// we need to record the value of the stack pointer on entry to the function.
-bool WebAssemblyFrameLowering::hasBP(
-    const MachineFunction &MF) const {
+bool WebAssemblyFrameLowering::hasBP(const MachineFunction &MF) const {
   const auto *RegInfo =
       MF.getSubtarget<WebAssemblySubtarget>().getRegisterInfo();
   return RegInfo->needsStackRealignment(MF);
@@ -78,36 +78,60 @@ bool WebAssemblyFrameLowering::hasReservedCallFrame(
   return !MF.getFrameInfo().hasVarSizedObjects();
 }
 
+// Returns true if this function needs a local user-space stack pointer for its
+// local frame (not for exception handling).
+bool WebAssemblyFrameLowering::needsSPForLocalFrame(
+    const MachineFunction &MF) const {
+  auto &MFI = MF.getFrameInfo();
+  return MFI.getStackSize() || MFI.adjustsStack() || hasFP(MF);
+}
+
+// In function with EH pads, we need to make a copy of the value of
+// __stack_pointer global in SP32 register, in order to use it when restoring
+// __stack_pointer after an exception is caught.
+bool WebAssemblyFrameLowering::needsPrologForEH(
+    const MachineFunction &MF) const {
+  auto EHType = MF.getTarget().getMCAsmInfo()->getExceptionHandlingType();
+  return EHType == ExceptionHandling::Wasm &&
+         MF.getFunction().hasPersonalityFn() && MF.getFrameInfo().hasCalls();
+}
 
 /// Returns true if this function needs a local user-space stack pointer.
 /// Unlike a machine stack pointer, the wasm user stack pointer is a global
 /// variable, so it is loaded into a register in the prolog.
-bool WebAssemblyFrameLowering::needsSP(const MachineFunction &MF,
-                                       const MachineFrameInfo &MFI) const {
-  return MFI.getStackSize() || MFI.adjustsStack() || hasFP(MF);
+bool WebAssemblyFrameLowering::needsSP(const MachineFunction &MF) const {
+  return needsSPForLocalFrame(MF) || needsPrologForEH(MF);
 }
 
 /// Returns true if the local user-space stack pointer needs to be written back
-/// to memory by this function (this is not meaningful if needsSP is false). If
-/// false, the stack red zone can be used and only a local SP is needed.
+/// to __stack_pointer global by this function (this is not meaningful if
+/// needsSP is false). If false, the stack red zone can be used and only a local
+/// SP is needed.
 bool WebAssemblyFrameLowering::needsSPWriteback(
-    const MachineFunction &MF, const MachineFrameInfo &MFI) const {
-  assert(needsSP(MF, MFI));
-  return MFI.getStackSize() > RedZoneSize || MFI.hasCalls() ||
-         MF.getFunction().hasFnAttribute(Attribute::NoRedZone);
+    const MachineFunction &MF) const {
+  auto &MFI = MF.getFrameInfo();
+  assert(needsSP(MF));
+  // When we don't need a local stack pointer for its local frame but only to
+  // support EH, we don't need to write SP back in the epilog, because we don't
+  // bump down the stack pointer in the prolog. We need to write SP back in the
+  // epilog only if
+  // 1. We need SP not only for EH support but also because we actually use
+  // stack or we have a frame address taken.
+  // 2. We cannot use the red zone.
+  bool CanUseRedZone = MFI.getStackSize() <= RedZoneSize && !MFI.hasCalls() &&
+                       !MF.getFunction().hasFnAttribute(Attribute::NoRedZone);
+  return needsSPForLocalFrame(MF) && !CanUseRedZone;
 }
 
-static void writeSPToMemory(unsigned SrcReg, MachineFunction &MF,
-                            MachineBasicBlock &MBB,
-                            MachineBasicBlock::iterator &InsertAddr,
-                            MachineBasicBlock::iterator &InsertStore,
-                            const DebugLoc &DL) {
+void WebAssemblyFrameLowering::writeSPToGlobal(
+    unsigned SrcReg, MachineFunction &MF, MachineBasicBlock &MBB,
+    MachineBasicBlock::iterator &InsertStore, const DebugLoc &DL) const {
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
 
   const char *ES = "__stack_pointer";
   auto *SPSymbol = MF.createExternalSymbolName(ES);
-  BuildMI(MBB, InsertStore, DL, TII->get(WebAssembly::SET_GLOBAL_I32))
-      .addExternalSymbol(SPSymbol)
+  BuildMI(MBB, InsertStore, DL, TII->get(WebAssembly::GLOBAL_SET_I32))
+      .addExternalSymbol(SPSymbol, WebAssemblyII::MO_SYMBOL_GLOBAL)
       .addReg(SrcReg);
 }
 
@@ -119,9 +143,9 @@ WebAssemblyFrameLowering::eliminateCallFramePseudoInstr(
          "Call frame pseudos should only be used for dynamic stack adjustment");
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   if (I->getOpcode() == TII->getCallFrameDestroyOpcode() &&
-      needsSPWriteback(MF, MF.getFrameInfo())) {
+      needsSPWriteback(MF)) {
     DebugLoc DL = I->getDebugLoc();
-    writeSPToMemory(WebAssembly::SP32, MF, MBB, I, I, DL);
+    writeSPToGlobal(WebAssembly::SP32, MF, MBB, I, DL);
   }
   return MBB.erase(I);
 }
@@ -133,7 +157,8 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
   assert(MFI.getCalleeSavedInfo().empty() &&
          "WebAssembly should not have callee-saved registers");
 
-  if (!needsSP(MF, MFI)) return;
+  if (!needsSP(MF))
+    return;
   uint64_t StackSize = MFI.getStackSize();
 
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
@@ -152,8 +177,8 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
 
   const char *ES = "__stack_pointer";
   auto *SPSymbol = MF.createExternalSymbolName(ES);
-  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::GET_GLOBAL_I32), SPReg)
-      .addExternalSymbol(SPSymbol);
+  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::GLOBAL_GET_I32), SPReg)
+      .addExternalSymbol(SPSymbol, WebAssemblyII::MO_SYMBOL_GLOBAL);
 
   bool HasBP = hasBP(MF);
   if (HasBP) {
@@ -177,7 +202,7 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
     unsigned BitmaskReg = MRI.createVirtualRegister(PtrRC);
     unsigned Alignment = MFI.getMaxAlignment();
     assert((1u << countTrailingZeros(Alignment)) == Alignment &&
-      "Alignment must be a power of 2");
+           "Alignment must be a power of 2");
     BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), BitmaskReg)
         .addImm((int)~(Alignment - 1));
     BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::AND_I32),
@@ -189,20 +214,19 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
     // Unlike most conventional targets (where FP points to the saved FP),
     // FP points to the bottom of the fixed-size locals, so we can use positive
     // offsets in load/store instructions.
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::COPY),
-            WebAssembly::FP32)
+    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::COPY), WebAssembly::FP32)
         .addReg(WebAssembly::SP32);
   }
-  if (StackSize && needsSPWriteback(MF, MFI)) {
-    writeSPToMemory(WebAssembly::SP32, MF, MBB, InsertPt, InsertPt, DL);
+  if (StackSize && needsSPWriteback(MF)) {
+    writeSPToGlobal(WebAssembly::SP32, MF, MBB, InsertPt, DL);
   }
 }
 
 void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
                                             MachineBasicBlock &MBB) const {
-  auto &MFI = MF.getFrameInfo();
-  uint64_t StackSize = MFI.getStackSize();
-  if (!needsSP(MF, MFI) || !needsSPWriteback(MF, MFI)) return;
+  uint64_t StackSize = MF.getFrameInfo().getStackSize();
+  if (!needsSP(MF) || !needsSPWriteback(MF))
+    return;
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   auto &MRI = MF.getRegInfo();
   auto InsertPt = MBB.getFirstTerminator();
@@ -214,7 +238,6 @@ void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
   // Restore the stack pointer. If we had fixed-size locals, add the offset
   // subtracted in the prolog.
   unsigned SPReg = 0;
-  MachineBasicBlock::iterator InsertAddr = InsertPt;
   if (hasBP(MF)) {
     auto FI = MF.getInfo<WebAssemblyFunctionInfo>();
     SPReg = FI->getBasePointerVreg();
@@ -222,9 +245,8 @@ void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
     const TargetRegisterClass *PtrRC =
         MRI.getTargetRegisterInfo()->getPointerRegClass(MF);
     unsigned OffsetReg = MRI.createVirtualRegister(PtrRC);
-    InsertAddr =
-        BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), OffsetReg)
-            .addImm(StackSize);
+    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), OffsetReg)
+        .addImm(StackSize);
     // In the epilog we don't need to write the result back to the SP32 physreg
     // because it won't be used again. We can use a stackified register instead.
     SPReg = MRI.createVirtualRegister(PtrRC);
@@ -235,5 +257,5 @@ void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
     SPReg = hasFP(MF) ? WebAssembly::FP32 : WebAssembly::SP32;
   }
 
-  writeSPToMemory(SPReg, MF, MBB, InsertAddr, InsertPt, DL);
+  writeSPToGlobal(SPReg, MF, MBB, InsertPt, DL);
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyFrameLowering.h b/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
index fe23e418a3f1..c6fa8261b03f 100644
--- a/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
+++ b/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
@@ -22,9 +22,10 @@ namespace llvm {
 class MachineFrameInfo;
 
 class WebAssemblyFrameLowering final : public TargetFrameLowering {
- public:
+public:
   /// Size of the red zone for the user stack (leaf functions can use this much
-  /// space below the stack pointer without writing it back to memory).
+  /// space below the stack pointer without writing it back to __stack_pointer
+  /// global).
   // TODO: (ABI) Revisit and decide how large it should be.
   static const size_t RedZoneSize = 128;
 
@@ -34,9 +35,9 @@ class WebAssemblyFrameLowering final : public TargetFrameLowering {
                             /*TransientStackAlignment=*/16,
                             /*StackRealignable=*/true) {}
 
-  MachineBasicBlock::iterator eliminateCallFramePseudoInstr(
-      MachineFunction &MF, MachineBasicBlock &MBB,
-      MachineBasicBlock::iterator I) const override;
+  MachineBasicBlock::iterator
+  eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator I) const override;
 
   /// These methods insert prolog and epilog code into the function.
   void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
@@ -45,13 +46,21 @@ class WebAssemblyFrameLowering final : public TargetFrameLowering {
   bool hasFP(const MachineFunction &MF) const override;
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
 
- private:
+  bool needsPrologForEH(const MachineFunction &MF) const;
+
+  /// Write SP back to __stack_pointer global.
+  void writeSPToGlobal(unsigned SrcReg, MachineFunction &MF,
+                       MachineBasicBlock &MBB,
+                       MachineBasicBlock::iterator &InsertStore,
+                       const DebugLoc &DL) const;
+
+private:
   bool hasBP(const MachineFunction &MF) const;
-  bool needsSP(const MachineFunction &MF, const MachineFrameInfo &MFI) const;
-  bool needsSPWriteback(const MachineFunction &MF,
-                        const MachineFrameInfo &MFI) const;
+  bool needsSPForLocalFrame(const MachineFunction &MF) const;
+  bool needsSP(const MachineFunction &MF) const;
+  bool needsSPWriteback(const MachineFunction &MF) const;
 };
 
-}  // end namespace llvm
+} // end namespace llvm
 
 #endif
diff --git a/lib/Target/WebAssembly/WebAssemblyISD.def b/lib/Target/WebAssembly/WebAssemblyISD.def
index c12550feabbb..e987d7f7f43a 100644
--- a/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -21,5 +21,10 @@ HANDLE_NODETYPE(ARGUMENT)
 HANDLE_NODETYPE(Wrapper)
 HANDLE_NODETYPE(BR_IF)
 HANDLE_NODETYPE(BR_TABLE)
+HANDLE_NODETYPE(SHUFFLE)
+HANDLE_NODETYPE(VEC_SHL)
+HANDLE_NODETYPE(VEC_SHR_S)
+HANDLE_NODETYPE(VEC_SHR_U)
+HANDLE_NODETYPE(THROW)
 
 // add memory opcodes starting at ISD::FIRST_TARGET_MEMORY_OPCODE here...
diff --git a/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index fdf3a30a5c0e..0a7464cedc90 100644
--- a/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -48,6 +48,10 @@ public:
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override {
+    LLVM_DEBUG(dbgs() << "********** ISelDAGToDAG **********\n"
+                         "********** Function: "
+                      << MF.getName() << '\n');
+
     ForCodeSize = MF.getFunction().hasFnAttribute(Attribute::OptimizeForSize) ||
                   MF.getFunction().hasFnAttribute(Attribute::MinSize);
     Subtarget = &MF.getSubtarget<WebAssemblySubtarget>();
diff --git a/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 283e703e1f6c..003848e34227 100644
--- a/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -21,8 +21,10 @@
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/WasmEHFuncInfo.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/Function.h"
@@ -42,6 +44,8 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
 
   // Booleans always contain 0 or 1.
   setBooleanContents(ZeroOrOneBooleanContent);
+  // Except in SIMD vectors
+  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
   // WebAssembly does not produce floating-point exceptions on normal floating
   // point operations.
   setHasFloatingPointExceptions(false);
@@ -60,6 +64,10 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
     addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
     addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
+    if (Subtarget->hasUnimplementedSIMD128()) {
+      addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
+      addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
+    }
   }
   // Compute derived properties from the register classes.
   computeRegisterProperties(Subtarget->getRegisterInfo());
@@ -77,7 +85,7 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
   setOperationAction(ISD::VAEND, MVT::Other, Expand);
 
-  for (auto T : {MVT::f32, MVT::f64}) {
+  for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
     // Don't expand the floating-point types to constant pools.
     setOperationAction(ISD::ConstantFP, T, Legal);
     // Expand floating-point comparisons.
@@ -85,17 +93,17 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
                     ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
       setCondCodeAction(CC, T, Expand);
     // Expand floating-point library function operators.
-    for (auto Op : {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM,
-                    ISD::FMA})
+    for (auto Op :
+         {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
       setOperationAction(Op, T, Expand);
     // Note supported floating-point library function operators that otherwise
     // default to expand.
     for (auto Op :
          {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
       setOperationAction(Op, T, Legal);
-    // Support minnan and maxnan, which otherwise default to expand.
-    setOperationAction(ISD::FMINNAN, T, Legal);
-    setOperationAction(ISD::FMAXNAN, T, Legal);
+    // Support minimum and maximum, which otherwise default to expand.
+    setOperationAction(ISD::FMINIMUM, T, Legal);
+    setOperationAction(ISD::FMAXIMUM, T, Legal);
     // WebAssembly currently has no builtin f16 support.
     setOperationAction(ISD::FP16_TO_FP, T, Expand);
     setOperationAction(ISD::FP_TO_FP16, T, Expand);
@@ -103,24 +111,75 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     setTruncStoreAction(T, MVT::f16, Expand);
   }
 
-  for (auto T : {MVT::i32, MVT::i64}) {
-    // Expand unavailable integer operations.
-    for (auto Op :
-         {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
-          ISD::MULHS, ISD::MULHU, ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS,
-          ISD::SRA_PARTS, ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC,
-          ISD::SUBE}) {
+  // Support saturating add for i8x16 and i16x8
+  if (Subtarget->hasSIMD128())
+    for (auto T : {MVT::v16i8, MVT::v8i16})
+      for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
+        setOperationAction(Op, T, Legal);
+
+  // Expand unavailable integer operations.
+  for (auto Op :
+       {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
+        ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
+        ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
+    for (auto T : {MVT::i32, MVT::i64}) {
       setOperationAction(Op, T, Expand);
     }
+    if (Subtarget->hasSIMD128()) {
+      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) {
+        setOperationAction(Op, T, Expand);
+      }
+      if (Subtarget->hasUnimplementedSIMD128()) {
+        setOperationAction(Op, MVT::v2i64, Expand);
+      }
+    }
+  }
+
+  // There is no i64x2.mul instruction
+  setOperationAction(ISD::MUL, MVT::v2i64, Expand);
+
+  // We have custom shuffle lowering to expose the shuffle mask
+  if (Subtarget->hasSIMD128()) {
+    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) {
+      setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
+    }
+    if (Subtarget->hasUnimplementedSIMD128()) {
+      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
+      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
+    }
+  }
+
+  // Custom lowering since wasm shifts must have a scalar shift amount
+  if (Subtarget->hasSIMD128()) {
+    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
+      for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
+        setOperationAction(Op, T, Custom);
+    if (Subtarget->hasUnimplementedSIMD128())
+      for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
+        setOperationAction(Op, MVT::v2i64, Custom);
   }
 
+  // There are no select instructions for vectors
+  if (Subtarget->hasSIMD128())
+    for (auto Op : {ISD::VSELECT, ISD::SELECT_CC, ISD::SELECT}) {
+      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
+        setOperationAction(Op, T, Expand);
+      if (Subtarget->hasUnimplementedSIMD128())
+        for (auto T : {MVT::v2i64, MVT::v2f64})
+          setOperationAction(Op, T, Expand);
+    }
+
   // As a special case, these operators use the type to mean the type to
   // sign-extend from.
   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
   if (!Subtarget->hasSignExt()) {
+    // Sign extends are legal only when extending a vector extract
+    auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
     for (auto T : {MVT::i8, MVT::i16, MVT::i32})
-      setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
+      setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
   }
+  for (auto T : MVT::integer_vector_valuetypes())
+    setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
 
   // Dynamic stack allocation: use the default expansion.
   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
@@ -142,21 +201,72 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   //  - Floating-point extending loads.
   //  - Floating-point truncating stores.
   //  - i1 extending loads.
+  //  - extending/truncating SIMD loads/stores
   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
   for (auto T : MVT::integer_valuetypes())
     for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
       setLoadExtAction(Ext, T, MVT::i1, Promote);
+  if (Subtarget->hasSIMD128()) {
+    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
+                   MVT::v2f64}) {
+      for (auto MemT : MVT::vector_valuetypes()) {
+        if (MVT(T) != MemT) {
+          setTruncStoreAction(T, MemT, Expand);
+          for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
+            setLoadExtAction(Ext, T, MemT, Expand);
+        }
+      }
+    }
+  }
+
+  // Expand additional SIMD ops that V8 hasn't implemented yet
+  if (Subtarget->hasSIMD128() && !Subtarget->hasUnimplementedSIMD128()) {
+    setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
+    setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
+  }
+
+  // Custom lower lane accesses to expand out variable indices
+  if (Subtarget->hasSIMD128()) {
+    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) {
+      setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
+      setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
+    }
+    if (Subtarget->hasUnimplementedSIMD128()) {
+      for (auto T : {MVT::v2i64, MVT::v2f64}) {
+        setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
+        setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
+      }
+    }
+  }
 
   // Trap lowers to wasm unreachable
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // Exception handling intrinsics
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
+  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
   setMaxAtomicSizeInBitsSupported(64);
 }
 
+TargetLowering::AtomicExpansionKind
+WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+  // We have wasm instructions for these
+  switch (AI->getOperation()) {
+  case AtomicRMWInst::Add:
+  case AtomicRMWInst::Sub:
+  case AtomicRMWInst::And:
+  case AtomicRMWInst::Or:
+  case AtomicRMWInst::Xor:
+  case AtomicRMWInst::Xchg:
+    return AtomicExpansionKind::None;
+  default:
+    break;
+  }
+  return AtomicExpansionKind::CmpXChg;
+}
+
 FastISel *WebAssemblyTargetLowering::createFastISel(
     FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
   return WebAssembly::createFastISel(FuncInfo, LibInfo);
@@ -171,7 +281,8 @@ bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
 MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
                                                       EVT VT) const {
   unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
-  if (BitWidth > 1 && BitWidth < 8) BitWidth = 8;
+  if (BitWidth > 1 && BitWidth < 8)
+    BitWidth = 8;
 
   if (BitWidth > 64) {
     // The shift will be lowered to a libcall, and compiler-rt libcalls expect
@@ -190,17 +301,11 @@ MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
 // Lower an fp-to-int conversion operator from the LLVM opcode, which has an
 // undefined result on invalid/overflow, to the WebAssembly opcode, which
 // traps on invalid/overflow.
-static MachineBasicBlock *
-LowerFPToInt(
-    MachineInstr &MI,
-    DebugLoc DL,
-    MachineBasicBlock *BB,
-    const TargetInstrInfo &TII,
-    bool IsUnsigned,
-    bool Int64,
-    bool Float64,
-    unsigned LoweredOpcode
-) {
+static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
+                                       MachineBasicBlock *BB,
+                                       const TargetInstrInfo &TII,
+                                       bool IsUnsigned, bool Int64,
+                                       bool Float64, unsigned LoweredOpcode) {
   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
   unsigned OutReg = MI.getOperand(0).getReg();
@@ -232,8 +337,7 @@ LowerFPToInt(
 
   // Transfer the remainder of BB and its successor edges to DoneMBB.
   DoneMBB->splice(DoneMBB->begin(), BB,
-                  std::next(MachineBasicBlock::iterator(MI)),
-                  BB->end());
+                  std::next(MachineBasicBlock::iterator(MI)), BB->end());
   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
 
   BB->addSuccessor(TrueMBB);
@@ -255,45 +359,33 @@ LowerFPToInt(
   if (IsUnsigned) {
     Tmp0 = InReg;
   } else {
-    BuildMI(BB, DL, TII.get(Abs), Tmp0)
-        .addReg(InReg);
+    BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
   }
   BuildMI(BB, DL, TII.get(FConst), Tmp1)
       .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
-  BuildMI(BB, DL, TII.get(LT), CmpReg)
-      .addReg(Tmp0)
-      .addReg(Tmp1);
+  BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
 
   // For unsigned numbers, we have to do a separate comparison with zero.
   if (IsUnsigned) {
     Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
-    unsigned SecondCmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+    unsigned SecondCmpReg =
+        MRI.createVirtualRegister(&WebAssembly::I32RegClass);
     unsigned AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
     BuildMI(BB, DL, TII.get(FConst), Tmp1)
         .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
-    BuildMI(BB, DL, TII.get(GE), SecondCmpReg)
-        .addReg(Tmp0)
-        .addReg(Tmp1);
-    BuildMI(BB, DL, TII.get(And), AndReg)
-        .addReg(CmpReg)
-        .addReg(SecondCmpReg);
+    BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
+    BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
     CmpReg = AndReg;
   }
 
-  BuildMI(BB, DL, TII.get(Eqz), EqzReg)
-      .addReg(CmpReg);
+  BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
 
   // Create the CFG diamond to select between doing the conversion or using
   // the substitute value.
-  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF))
-      .addMBB(TrueMBB)
-      .addReg(EqzReg);
-  BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg)
-      .addReg(InReg);
-  BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR))
-      .addMBB(DoneMBB);
-  BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg)
-      .addImm(Substitute);
+  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
+  BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
+  BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
+  BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
   BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
       .addReg(FalseReg)
       .addMBB(FalseMBB)
@@ -303,16 +395,14 @@ LowerFPToInt(
   return DoneMBB;
 }
 
-MachineBasicBlock *
-WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
-    MachineInstr &MI,
-    MachineBasicBlock *BB
-) const {
+MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
+    MachineInstr &MI, MachineBasicBlock *BB) const {
   const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
   DebugLoc DL = MI.getDebugLoc();
 
   switch (MI.getOpcode()) {
-  default: llvm_unreachable("Unexpected instr type to insert");
+  default:
+    llvm_unreachable("Unexpected instr type to insert");
   case WebAssembly::FP_TO_SINT_I32_F32:
     return LowerFPToInt(MI, DL, BB, TII, false, false, false,
                         WebAssembly::I32_TRUNC_S_F32);
@@ -337,17 +427,17 @@ WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
   case WebAssembly::FP_TO_UINT_I64_F64:
     return LowerFPToInt(MI, DL, BB, TII, true, true, true,
                         WebAssembly::I64_TRUNC_U_F64);
-  llvm_unreachable("Unexpected instruction to emit with custom inserter");
+    llvm_unreachable("Unexpected instruction to emit with custom inserter");
   }
 }
 
-const char *WebAssemblyTargetLowering::getTargetNodeName(
-    unsigned Opcode) const {
+const char *
+WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
   switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
-    case WebAssemblyISD::FIRST_NUMBER:
-      break;
-#define HANDLE_NODETYPE(NODE) \
-  case WebAssemblyISD::NODE:  \
+  case WebAssemblyISD::FIRST_NUMBER:
+    break;
+#define HANDLE_NODETYPE(NODE)                                                  \
+  case WebAssemblyISD::NODE:                                                   \
     return "WebAssemblyISD::" #NODE;
 #include "WebAssemblyISD.def"
 #undef HANDLE_NODETYPE
@@ -362,21 +452,21 @@ WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
   // WebAssembly register class.
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
-      case 'r':
-        assert(VT != MVT::iPTR && "Pointer MVT not expected here");
-        if (Subtarget->hasSIMD128() && VT.isVector()) {
-          if (VT.getSizeInBits() == 128)
-            return std::make_pair(0U, &WebAssembly::V128RegClass);
-        }
-        if (VT.isInteger() && !VT.isVector()) {
-          if (VT.getSizeInBits() <= 32)
-            return std::make_pair(0U, &WebAssembly::I32RegClass);
-          if (VT.getSizeInBits() <= 64)
-            return std::make_pair(0U, &WebAssembly::I64RegClass);
-        }
-        break;
-      default:
-        break;
+    case 'r':
+      assert(VT != MVT::iPTR && "Pointer MVT not expected here");
+      if (Subtarget->hasSIMD128() && VT.isVector()) {
+        if (VT.getSizeInBits() == 128)
+          return std::make_pair(0U, &WebAssembly::V128RegClass);
+      }
+      if (VT.isInteger() && !VT.isVector()) {
+        if (VT.getSizeInBits() <= 32)
+          return std::make_pair(0U, &WebAssembly::I32RegClass);
+        if (VT.getSizeInBits() <= 64)
+          return std::make_pair(0U, &WebAssembly::I64RegClass);
+      }
+      break;
+    default:
+      break;
     }
   }
 
@@ -395,16 +485,17 @@ bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
 
 bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                                       const AddrMode &AM,
-                                                      Type *Ty,
-                                                      unsigned AS,
+                                                      Type *Ty, unsigned AS,
                                                       Instruction *I) const {
   // WebAssembly offsets are added as unsigned without wrapping. The
   // isLegalAddressingMode gives us no way to determine if wrapping could be
   // happening, so we approximate this by accepting only non-negative offsets.
-  if (AM.BaseOffs < 0) return false;
+  if (AM.BaseOffs < 0)
+    return false;
 
   // WebAssembly has no scale register operands.
-  if (AM.Scale != 0) return false;
+  if (AM.Scale != 0)
+    return false;
 
   // Everything else is legal.
   return true;
@@ -418,7 +509,8 @@ bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
   // for the kinds of things that LLVM uses this for (merging adjacent stores
   // of constants, etc.), WebAssembly implementations will either want the
   // unaligned access or they'll split anyway.
-  if (Fast) *Fast = true;
+  if (Fast)
+    *Fast = true;
   return true;
 }
 
@@ -438,6 +530,46 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
   return TargetLowering::getSetCCResultType(DL, C, VT);
 }
 
+bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
+                                                   const CallInst &I,
+                                                   MachineFunction &MF,
+                                                   unsigned Intrinsic) const {
+  switch (Intrinsic) {
+  case Intrinsic::wasm_atomic_notify:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i32;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = 4;
+    // atomic.notify instruction does not really load the memory specified with
+    // this argument, but MachineMemOperand should either be load or store, so
+    // we set this to a load.
+    // FIXME Volatile isn't really correct, but currently all LLVM atomic
+    // instructions are treated as volatiles in the backend, so we should be
+    // consistent. The same applies for wasm_atomic_wait intrinsics too.
+    Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
+    return true;
+  case Intrinsic::wasm_atomic_wait_i32:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i32;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = 4;
+    Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
+    return true;
+  case Intrinsic::wasm_atomic_wait_i64:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i64;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = 8;
+    Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
+    return true;
+  default:
+    return false;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // WebAssembly Lowering private implementation.
 //===----------------------------------------------------------------------===//
@@ -465,8 +597,9 @@ static bool CallingConvSupported(CallingConv::ID CallConv) {
          CallConv == CallingConv::CXX_FAST_TLS;
 }
 
-SDValue WebAssemblyTargetLowering::LowerCall(
-    CallLoweringInfo &CLI, SmallVectorImpl<SDValue> &InVals) const {
+SDValue
+WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
+                                     SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG = CLI.DAG;
   SDLoc DL = CLI.DL;
   SDValue Chain = CLI.Chain;
@@ -568,9 +701,9 @@ SDValue WebAssemblyTargetLowering::LowerCall(
       FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
       SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
                                 DAG.getConstant(Offset, DL, PtrVT));
-      Chains.push_back(DAG.getStore(
-          Chain, DL, Arg, Add,
-          MachinePointerInfo::getFixedStack(MF, FI, Offset), 0));
+      Chains.push_back(
+          DAG.getStore(Chain, DL, Arg, Add,
+                       MachinePointerInfo::getFixedStack(MF, FI, Offset), 0));
     }
     if (!Chains.empty())
       Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
@@ -588,7 +721,8 @@ SDValue WebAssemblyTargetLowering::LowerCall(
   Ops.append(OutVals.begin(),
              IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
   // Add a pointer to the vararg buffer.
-  if (IsVarArg) Ops.push_back(FINode);
+  if (IsVarArg)
+    Ops.push_back(FINode);
 
   SmallVector<EVT, 8> InTys;
   for (const auto &In : Ins) {
@@ -682,11 +816,10 @@ SDValue WebAssemblyTargetLowering::LowerFormalArguments(
       fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
     // Ignore In.getOrigAlign() because all our arguments are passed in
     // registers.
-    InVals.push_back(
-        In.Used
-            ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
-                          DAG.getTargetConstant(InVals.size(), DL, MVT::i32))
-            : DAG.getUNDEF(In.VT));
+    InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
+                                           DAG.getTargetConstant(InVals.size(),
+                                                                 DL, MVT::i32))
+                             : DAG.getUNDEF(In.VT));
 
     // Record the number and types of arguments.
     MFI->addParam(In.VT);
@@ -706,12 +839,18 @@ SDValue WebAssemblyTargetLowering::LowerFormalArguments(
     MFI->addParam(PtrVT);
   }
 
-  // Record the number and types of results.
+  // Record the number and types of arguments and results.
   SmallVector<MVT, 4> Params;
   SmallVector<MVT, 4> Results;
-  ComputeSignatureVTs(MF.getFunction(), DAG.getTarget(), Params, Results);
+  ComputeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(),
+                      DAG.getTarget(), Params, Results);
   for (MVT VT : Results)
     MFI->addResult(VT);
+  // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
+  // the param logic here with ComputeSignatureVTs
+  assert(MFI->getParams().size() == Params.size() &&
+         std::equal(MFI->getParams().begin(), MFI->getParams().end(),
+                    Params.begin()));
 
   return Chain;
 }
@@ -724,34 +863,47 @@ SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SDLoc DL(Op);
   switch (Op.getOpcode()) {
-    default:
-      llvm_unreachable("unimplemented operation lowering");
-      return SDValue();
-    case ISD::FrameIndex:
-      return LowerFrameIndex(Op, DAG);
-    case ISD::GlobalAddress:
-      return LowerGlobalAddress(Op, DAG);
-    case ISD::ExternalSymbol:
-      return LowerExternalSymbol(Op, DAG);
-    case ISD::JumpTable:
-      return LowerJumpTable(Op, DAG);
-    case ISD::BR_JT:
-      return LowerBR_JT(Op, DAG);
-    case ISD::VASTART:
-      return LowerVASTART(Op, DAG);
-    case ISD::BlockAddress:
-    case ISD::BRIND:
-      fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
-      return SDValue();
-    case ISD::RETURNADDR: // Probably nothing meaningful can be returned here.
-      fail(DL, DAG, "WebAssembly hasn't implemented __builtin_return_address");
-      return SDValue();
-    case ISD::FRAMEADDR:
-      return LowerFRAMEADDR(Op, DAG);
-    case ISD::CopyToReg:
-      return LowerCopyToReg(Op, DAG);
-    case ISD::INTRINSIC_WO_CHAIN:
-      return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+  default:
+    llvm_unreachable("unimplemented operation lowering");
+    return SDValue();
+  case ISD::FrameIndex:
+    return LowerFrameIndex(Op, DAG);
+  case ISD::GlobalAddress:
+    return LowerGlobalAddress(Op, DAG);
+  case ISD::ExternalSymbol:
+    return LowerExternalSymbol(Op, DAG);
+  case ISD::JumpTable:
+    return LowerJumpTable(Op, DAG);
+  case ISD::BR_JT:
+    return LowerBR_JT(Op, DAG);
+  case ISD::VASTART:
+    return LowerVASTART(Op, DAG);
+  case ISD::BlockAddress:
+  case ISD::BRIND:
+    fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
+    return SDValue();
+  case ISD::RETURNADDR: // Probably nothing meaningful can be returned here.
+    fail(DL, DAG, "WebAssembly hasn't implemented __builtin_return_address");
+    return SDValue();
+  case ISD::FRAMEADDR:
+    return LowerFRAMEADDR(Op, DAG);
+  case ISD::CopyToReg:
+    return LowerCopyToReg(Op, DAG);
+  case ISD::INTRINSIC_WO_CHAIN:
+    return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+  case ISD::EXTRACT_VECTOR_ELT:
+  case ISD::INSERT_VECTOR_ELT:
+    return LowerAccessVectorElement(Op, DAG);
+  case ISD::INTRINSIC_VOID:
+    return LowerINTRINSIC_VOID(Op, DAG);
+  case ISD::SIGN_EXTEND_INREG:
+    return LowerSIGN_EXTEND_INREG(Op, DAG);
+  case ISD::VECTOR_SHUFFLE:
+    return LowerVECTOR_SHUFFLE(Op, DAG);
+  case ISD::SHL:
+  case ISD::SRA:
+  case ISD::SRL:
+    return LowerShift(Op, DAG);
   }
 }
 
@@ -763,21 +915,20 @@ SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
     // the FI to some LEA-like instruction, but since we don't have that, we
     // need to insert some kind of instruction that can take an FI operand and
     // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
-    // copy_local between Op and its FI operand.
+    // local.copy between Op and its FI operand.
     SDValue Chain = Op.getOperand(0);
     SDLoc DL(Op);
     unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
     EVT VT = Src.getValueType();
-    SDValue Copy(
-        DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
-                                          : WebAssembly::COPY_I64,
-                           DL, VT, Src),
-        0);
+    SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
+                                                   : WebAssembly::COPY_I64,
+                                    DL, VT, Src),
+                 0);
     return Op.getNode()->getNumValues() == 1
                ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
-               : DAG.getCopyToReg(Chain, DL, Reg, Copy, Op.getNumOperands() == 4
-                                                            ? Op.getOperand(3)
-                                                            : SDValue());
+               : DAG.getCopyToReg(Chain, DL, Reg, Copy,
+                                  Op.getNumOperands() == 4 ? Op.getOperand(3)
+                                                           : SDValue());
   }
   return SDValue();
 }
@@ -817,8 +968,9 @@ SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
       DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset()));
 }
 
-SDValue WebAssemblyTargetLowering::LowerExternalSymbol(
-    SDValue Op, SelectionDAG &DAG) const {
+SDValue
+WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
+                                               SelectionDAG &DAG) const {
   SDLoc DL(Op);
   const auto *ES = cast<ExternalSymbolSDNode>(Op);
   EVT VT = Op.getValueType();
@@ -829,9 +981,10 @@ SDValue WebAssemblyTargetLowering::LowerExternalSymbol(
   // we don't know anything about the symbol other than its name, because all
   // external symbols used in target-independent SelectionDAG code are for
   // functions.
-  return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
-                     DAG.getTargetExternalSymbol(ES->getSymbol(), VT,
-                                                 /*TargetFlags=*/0x1));
+  return DAG.getNode(
+      WebAssemblyISD::Wrapper, DL, VT,
+      DAG.getTargetExternalSymbol(ES->getSymbol(), VT,
+                                  WebAssemblyII::MO_SYMBOL_FUNCTION));
 }
 
 SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
@@ -860,7 +1013,8 @@ SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
   const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
 
   // Add an operand for each case.
-  for (auto MBB : MBBs) Ops.push_back(DAG.getBasicBlock(MBB));
+  for (auto MBB : MBBs)
+    Ops.push_back(DAG.getBasicBlock(MBB));
 
   // TODO: For now, we just pick something arbitrary for a default case for now.
   // We really want to sniff out the guard and put in the real default case (and
@@ -893,10 +1047,181 @@ WebAssemblyTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   default:
     return {}; // Don't custom lower most intrinsics.
 
-  case Intrinsic::wasm_lsda:
-    // TODO For now, just return 0 not to crash
-    return DAG.getConstant(0, DL, Op.getValueType());
+  case Intrinsic::wasm_lsda: {
+    MachineFunction &MF = DAG.getMachineFunction();
+    EVT VT = Op.getValueType();
+    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    auto &Context = MF.getMMI().getContext();
+    MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
+                                            Twine(MF.getFunctionNumber()));
+    return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
+                       DAG.getMCSymbol(S, PtrVT));
+  }
+  }
+}
+
+SDValue
+WebAssemblyTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
+  SDLoc DL(Op);
+
+  switch (IntNo) {
+  default:
+    return {}; // Don't custom lower most intrinsics.
+
+  case Intrinsic::wasm_throw: {
+    int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
+    switch (Tag) {
+    case CPP_EXCEPTION: {
+      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+      MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+      const char *SymName = MF.createExternalSymbolName("__cpp_exception");
+      SDValue SymNode =
+          DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
+                      DAG.getTargetExternalSymbol(
+                          SymName, PtrVT, WebAssemblyII::MO_SYMBOL_EVENT));
+      return DAG.getNode(WebAssemblyISD::THROW, DL,
+                         MVT::Other, // outchain type
+                         {
+                             Op.getOperand(0), // inchain
+                             SymNode,          // exception symbol
+                             Op.getOperand(3)  // thrown value
+                         });
+    }
+    default:
+      llvm_unreachable("Invalid tag!");
+    }
+    break;
+  }
+  }
+}
+
+SDValue
+WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  // If sign extension operations are disabled, allow sext_inreg only if operand
+  // is a vector extract. SIMD does not depend on sign extension operations, but
+  // allowing sext_inreg in this context lets us have simple patterns to select
+  // extract_lane_s instructions. Expanding sext_inreg everywhere would be
+  // simpler in this file, but would necessitate large and brittle patterns to
+  // undo the expansion and select extract_lane_s instructions.
+  assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
+  if (Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT)
+    return Op;
+  // Otherwise expand
+  return SDValue();
+}
+
+SDValue
+WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
+  MVT VecType = Op.getOperand(0).getSimpleValueType();
+  assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
+  size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
+
+  // Space for two vector args and sixteen mask indices
+  SDValue Ops[18];
+  size_t OpIdx = 0;
+  Ops[OpIdx++] = Op.getOperand(0);
+  Ops[OpIdx++] = Op.getOperand(1);
+
+  // Expand mask indices to byte indices and materialize them as operands
+  for (size_t I = 0, Lanes = Mask.size(); I < Lanes; ++I) {
+    for (size_t J = 0; J < LaneBytes; ++J) {
+      // Lower undefs (represented by -1 in mask) to zero
+      uint64_t ByteIndex =
+          Mask[I] == -1 ? 0 : (uint64_t)Mask[I] * LaneBytes + J;
+      Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
+    }
+  }
+
+  return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
+}
+
+SDValue
+WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  // Allow constant lane indices, expand variable lane indices
+  SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
+  if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
+    return Op;
+  else
+    // Perform default expansion
+    return SDValue();
+}
+
+static SDValue UnrollVectorShift(SDValue Op, SelectionDAG &DAG) {
+  EVT LaneT = Op.getSimpleValueType().getVectorElementType();
+  // 32-bit and 64-bit unrolled shifts will have proper semantics
+  if (LaneT.bitsGE(MVT::i32))
+    return DAG.UnrollVectorOp(Op.getNode());
+  // Otherwise mask the shift value to get proper semantics from 32-bit shift
+  SDLoc DL(Op);
+  SDValue ShiftVal = Op.getOperand(1);
+  uint64_t MaskVal = LaneT.getSizeInBits() - 1;
+  SDValue MaskedShiftVal = DAG.getNode(
+      ISD::AND,                    // mask opcode
+      DL, ShiftVal.getValueType(), // masked value type
+      ShiftVal,                    // original shift value operand
+      DAG.getConstant(MaskVal, DL, ShiftVal.getValueType()) // mask operand
+  );
+
+  return DAG.UnrollVectorOp(
+      DAG.getNode(Op.getOpcode(),        // original shift opcode
+                  DL, Op.getValueType(), // original return type
+                  Op.getOperand(0),      // original vector operand,
+                  MaskedShiftVal         // new masked shift value operand
+                  )
+          .getNode());
+}
+
+SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
+                                              SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+
+  // Only manually lower vector shifts
+  assert(Op.getSimpleValueType().isVector());
+
+  // Expand all vector shifts until V8 fixes its implementation
+  // TODO: remove this once V8 is fixed
+  if (!Subtarget->hasUnimplementedSIMD128())
+    return UnrollVectorShift(Op, DAG);
+
+  // Unroll non-splat vector shifts
+  BuildVectorSDNode *ShiftVec;
+  SDValue SplatVal;
+  if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) ||
+      !(SplatVal = ShiftVec->getSplatValue()))
+    return UnrollVectorShift(Op, DAG);
+
+  // All splats except i64x2 const splats are handled by patterns
+  ConstantSDNode *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
+  if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64)
+    return Op;
+
+  // i64x2 const splats are custom lowered to avoid unnecessary wraps
+  unsigned Opcode;
+  switch (Op.getOpcode()) {
+  case ISD::SHL:
+    Opcode = WebAssemblyISD::VEC_SHL;
+    break;
+  case ISD::SRA:
+    Opcode = WebAssemblyISD::VEC_SHR_S;
+    break;
+  case ISD::SRL:
+    Opcode = WebAssemblyISD::VEC_SHR_U;
+    break;
+  default:
+    llvm_unreachable("unexpected opcode");
   }
+  APInt Shift = SplatConst->getAPIntValue().zextOrTrunc(32);
+  return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0),
+                     DAG.getConstant(Shift, DL, MVT::i32));
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index 79819493ac6a..59f4230ed889 100644
--- a/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -29,21 +29,22 @@ enum NodeType : unsigned {
 #undef HANDLE_NODETYPE
 };
 
-}  // end namespace WebAssemblyISD
+} // end namespace WebAssemblyISD
 
 class WebAssemblySubtarget;
 class WebAssemblyTargetMachine;
 
 class WebAssemblyTargetLowering final : public TargetLowering {
- public:
+public:
   WebAssemblyTargetLowering(const TargetMachine &TM,
                             const WebAssemblySubtarget &STI);
 
- private:
+private:
   /// Keep a pointer to the WebAssemblySubtarget around so that we can make the
   /// right decision when generating code for different targets.
   const WebAssemblySubtarget *Subtarget;
 
+  AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
   FastISel *createFastISel(FunctionLoweringInfo &FuncInfo,
                            const TargetLibraryInfo *LibInfo) const override;
   bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
@@ -52,9 +53,9 @@ class WebAssemblyTargetLowering final : public TargetLowering {
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *MBB) const override;
   const char *getTargetNodeName(unsigned Opcode) const override;
-  std::pair<unsigned, const TargetRegisterClass *> getRegForInlineAsmConstraint(
-      const TargetRegisterInfo *TRI, StringRef Constraint,
-      MVT VT) const override;
+  std::pair<unsigned, const TargetRegisterClass *>
+  getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
+                               StringRef Constraint, MVT VT) const override;
   bool isCheapToSpeculateCttz() const override;
   bool isCheapToSpeculateCtlz() const override;
   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
@@ -66,6 +67,9 @@ class WebAssemblyTargetLowering final : public TargetLowering {
 
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                          EVT VT) const override;
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+                          MachineFunction &MF,
+                          unsigned Intrinsic) const override;
 
   SDValue LowerCall(CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
@@ -94,13 +98,18 @@ class WebAssemblyTargetLowering final : public TargetLowering {
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCopyToReg(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerAccessVectorElement(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerShift(SDValue Op, SelectionDAG &DAG) const;
 };
 
 namespace WebAssembly {
 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
                          const TargetLibraryInfo *libInfo);
-}  // end namespace WebAssembly
+} // end namespace WebAssembly
 
-}  // end namespace llvm
+} // end namespace llvm
 
 #endif
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
index d879932b3232..5fb8ef90bc43 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
@@ -16,10 +16,16 @@
 // Atomic loads
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
+multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
+                    list<dag> pattern_r, string asmstr_r = "",
+                    string asmstr_s = "", bits<32> inst = -1> {
+  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
+              inst>,
+            Requires<[HasAtomics]>;
+}
+
 defm ATOMIC_LOAD_I32 : WebAssemblyLoad<I32, "i32.atomic.load", 0xfe10>;
 defm ATOMIC_LOAD_I64 : WebAssemblyLoad<I64, "i64.atomic.load", 0xfe11>;
-} // Defs = [ARGUMENTS]
 
 // Select loads with no constant offset.
 let Predicates = [HasAtomics] in {
@@ -54,13 +60,11 @@ def : LoadPatExternSymOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>;
 
 // Extending loads. Note that there are only zero-extending atomic loads, no
 // sign-extending loads.
-let Defs = [ARGUMENTS] in {
 defm ATOMIC_LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load8_u", 0xfe12>;
 defm ATOMIC_LOAD16_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load16_u", 0xfe13>;
 defm ATOMIC_LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load8_u", 0xfe14>;
 defm ATOMIC_LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load16_u", 0xfe15>;
 defm ATOMIC_LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load32_u", 0xfe16>;
-} // Defs = [ARGUMENTS]
 
 // Fragments for extending loads. These are different from regular loads because
 // the SDNodes are derived from AtomicSDNode rather than LoadSDNode and
@@ -110,7 +114,7 @@ def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
 def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i32
+// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s
 
 // Zero-extending loads with constant offset
 def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>;
@@ -192,10 +196,8 @@ def : LoadPatExternSymOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 // Atomic stores
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
 defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>;
 defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>;
-} // Defs = [ARGUMENTS]
 
 // We need an 'atomic' version of store patterns because store and atomic_store
 // nodes have different operand orders:
@@ -255,13 +257,11 @@ def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
 } // Predicates = [HasAtomics]
 
 // Truncating stores.
-let Defs = [ARGUMENTS] in {
 defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>;
 defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>;
 defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>;
 defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>;
 defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>;
-} // Defs = [ARGUMENTS]
 
 // Fragments for truncating stores.
 
@@ -333,8 +333,6 @@ def : AStorePatExternSymOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
 // Atomic binary read-modify-writes
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
-
 multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> {
   defm "" : I<(outs rc:$dst),
               (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val),
@@ -346,83 +344,82 @@ multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> {
 defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0xfe1e>;
 defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0xfe1f>;
 defm ATOMIC_RMW8_U_ADD_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.add", 0xfe20>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0xfe20>;
 defm ATOMIC_RMW16_U_ADD_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.add", 0xfe21>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0xfe21>;
 defm ATOMIC_RMW8_U_ADD_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.add", 0xfe22>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0xfe22>;
 defm ATOMIC_RMW16_U_ADD_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.add", 0xfe23>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0xfe23>;
 defm ATOMIC_RMW32_U_ADD_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.add", 0xfe24>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0xfe24>;
 
 defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0xfe25>;
 defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0xfe26>;
 defm ATOMIC_RMW8_U_SUB_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.sub", 0xfe27>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0xfe27>;
 defm ATOMIC_RMW16_U_SUB_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.sub", 0xfe28>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0xfe28>;
 defm ATOMIC_RMW8_U_SUB_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.sub", 0xfe29>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0xfe29>;
 defm ATOMIC_RMW16_U_SUB_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.sub", 0xfe2a>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0xfe2a>;
 defm ATOMIC_RMW32_U_SUB_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.sub", 0xfe2b>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0xfe2b>;
 
 defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0xfe2c>;
 defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0xfe2d>;
 defm ATOMIC_RMW8_U_AND_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.and", 0xfe2e>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0xfe2e>;
 defm ATOMIC_RMW16_U_AND_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.and", 0xfe2f>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0xfe2f>;
 defm ATOMIC_RMW8_U_AND_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.and", 0xfe30>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0xfe30>;
 defm ATOMIC_RMW16_U_AND_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.and", 0xfe31>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0xfe31>;
 defm ATOMIC_RMW32_U_AND_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.and", 0xfe32>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0xfe32>;
 
 defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0xfe33>;
 defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0xfe34>;
 defm ATOMIC_RMW8_U_OR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.or", 0xfe35>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0xfe35>;
 defm ATOMIC_RMW16_U_OR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.or", 0xfe36>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0xfe36>;
 defm ATOMIC_RMW8_U_OR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.or", 0xfe37>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0xfe37>;
 defm ATOMIC_RMW16_U_OR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.or", 0xfe38>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0xfe38>;
 defm ATOMIC_RMW32_U_OR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.or", 0xfe39>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0xfe39>;
 
 defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0xfe3a>;
 defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0xfe3b>;
 defm ATOMIC_RMW8_U_XOR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xor", 0xfe3c>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0xfe3c>;
 defm ATOMIC_RMW16_U_XOR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xor", 0xfe3d>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0xfe3d>;
 defm ATOMIC_RMW8_U_XOR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xor", 0xfe3e>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0xfe3e>;
 defm ATOMIC_RMW16_U_XOR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xor", 0xfe3f>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0xfe3f>;
 defm ATOMIC_RMW32_U_XOR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xor", 0xfe40>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0xfe40>;
 
 defm ATOMIC_RMW_XCHG_I32 :
   WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0xfe41>;
 defm ATOMIC_RMW_XCHG_I64 :
   WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0xfe42>;
 defm ATOMIC_RMW8_U_XCHG_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xchg", 0xfe43>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0xfe43>;
 defm ATOMIC_RMW16_U_XCHG_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xchg", 0xfe44>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0xfe44>;
 defm ATOMIC_RMW8_U_XCHG_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xchg", 0xfe45>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0xfe45>;
 defm ATOMIC_RMW16_U_XCHG_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xchg", 0xfe46>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0xfe46>;
 defm ATOMIC_RMW32_U_XCHG_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xchg", 0xfe47>;
-}
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0xfe47>;
 
 // Select binary RMWs with no constant offset.
 class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> :
@@ -533,7 +530,7 @@ class sext_bin_rmw_8_64<PatFrag kind> :
   PatFrag<(ops node:$addr, node:$val),
           (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>;
 class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>;
-// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32
+// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s
 
 // Patterns for various addressing modes for truncating-extending binary RMWs.
 multiclass BinRMWTruncExtPattern<
@@ -655,3 +652,368 @@ defm : BinRMWTruncExtPattern<
   ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32,
   ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>;
 } // Predicates = [HasAtomics]
+
+//===----------------------------------------------------------------------===//
+// Atomic ternary read-modify-writes
+//===----------------------------------------------------------------------===//
+
+// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success
+// flag}. When we use the success flag or both values, we can't make use of i64
+// truncate/extend versions of instructions for now, which is suboptimal.
+// Consider adding a pass after instruction selection that optimizes this case
+// if it is frequent.
+
+multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string Name, int Opcode> {
+  defm "" : I<(outs rc:$dst),
+              (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp,
+                   rc:$new),
+              (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+              !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new"),
+              !strconcat(Name, "\t${off}, ${p2align}"), Opcode>;
+}
+
+defm ATOMIC_RMW_CMPXCHG_I32 :
+  WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0xfe48>;
+defm ATOMIC_RMW_CMPXCHG_I64 :
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0xfe49>;
+defm ATOMIC_RMW8_U_CMPXCHG_I32 :
+  WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0xfe4a>;
+defm ATOMIC_RMW16_U_CMPXCHG_I32 :
+  WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0xfe4b>;
+defm ATOMIC_RMW8_U_CMPXCHG_I64 :
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0xfe4c>;
+defm ATOMIC_RMW16_U_CMPXCHG_I64 :
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0xfe4d>;
+defm ATOMIC_RMW32_U_CMPXCHG_I64 :
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0xfe4e>;
+
+// Select ternary RMWs with no constant offset.
+class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)),
+      (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>;
+
+// Select ternary RMWs with a constant offset.
+
+// Pattern with address + immediate offset
+class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> :
+  Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)),
+      (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>;
+
+class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
+                ty:$exp, ty:$new)),
+      (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>;
+
+class TerRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
+                ty:$exp, ty:$new)),
+      (inst 0, texternalsym:$off, I32:$addr, ty:$exp, ty:$new)>;
+
+// Select ternary RMWs with just a constant offset.
+class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind imm:$off, ty:$exp, ty:$new)),
+      (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>;
+
+class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)),
+      (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>;
+
+class TerRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, ty:$new)),
+      (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, ty:$new)>;
+
+// Patterns for various addressing modes.
+multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32,
+                         NI inst_64> {
+  def : TerRMWPatNoOffset<i32, rmw_32, inst_32>;
+  def : TerRMWPatNoOffset<i64, rmw_64, inst_64>;
+
+  def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>;
+  def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>;
+  def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>;
+  def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>;
+
+  def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>;
+  def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>;
+
+  def : TerRMWPatExternalSym<i32, rmw_32, inst_32>;
+  def : TerRMWPatExternalSym<i64, rmw_64, inst_64>;
+
+  def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>;
+  def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>;
+
+  def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>;
+  def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>;
+
+  def : TerRMWPatExternSymOffOnly<i32, rmw_32, inst_32>;
+  def : TerRMWPatExternSymOffOnly<i64, rmw_64, inst_64>;
+}
+
+let Predicates = [HasAtomics] in {
+defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64,
+                     ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>;
+} // Predicates = [HasAtomics]
+
+// Truncating & zero-extending ternary RMW patterns.
+// DAG legalization & optimization before instruction selection may introduce
+// additional nodes such as anyext or assertzext depending on operand types.
+class zext_ter_rmw_8_32<PatFrag kind> :
+  PatFrag<(ops node:$addr, node:$exp, node:$new),
+          (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>;
+class zext_ter_rmw_16_32<PatFrag kind> :
+  PatFrag<(ops node:$addr, node:$exp, node:$new),
+          (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>;
+class zext_ter_rmw_8_64<PatFrag kind> :
+  PatFrag<(ops node:$addr, node:$exp, node:$new),
+          (zext (i32 (assertzext (i32 (kind node:$addr,
+                                            (i32 (trunc (i64 node:$exp))),
+                                            (i32 (trunc (i64 node:$new))))))))>;
+class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>;
+class zext_ter_rmw_32_64<PatFrag kind> :
+  PatFrag<(ops node:$addr, node:$exp, node:$new),
+          (zext (i32 (kind node:$addr,
+                           (i32 (trunc (i64 node:$exp))),
+                           (i32 (trunc (i64 node:$new))))))>;
+
+// Truncating & sign-extending ternary RMW patterns.
+// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a
+// zext RMW; the next instruction will be sext_inreg which is selected by
+// itself.
+class sext_ter_rmw_8_32<PatFrag kind> :
+  PatFrag<(ops node:$addr, node:$exp, node:$new),
+          (kind node:$addr, node:$exp, node:$new)>;
+class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>;
+class sext_ter_rmw_8_64<PatFrag kind> :
+  PatFrag<(ops node:$addr, node:$exp, node:$new),
+          (anyext (i32 (assertzext (i32
+            (kind node:$addr,
+                  (i32 (trunc (i64 node:$exp))),
+                  (i32 (trunc (i64 node:$new))))))))>;
+class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>;
+// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s
+
+// Patterns for various addressing modes for truncating-extending ternary RMWs.
+multiclass TerRMWTruncExtPattern<
+  PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64,
+  NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> {
+  // Truncating-extending ternary RMWs with no constant offset
+  def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
+  def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
+
+  def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
+
+  // Truncating-extending ternary RMWs with a constant offset
+  def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>;
+  def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>;
+  def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>;
+  def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>;
+  def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>;
+  def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>;
+  def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>;
+  def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
+  def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
+  def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>;
+
+  def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>;
+  def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>;
+  def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>;
+  def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>;
+  def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>;
+  def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>;
+  def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
+  def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
+
+  def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
+  def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
+
+  def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
+
+  def : TerRMWPatExternalSym<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatExternalSym<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatExternalSym<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatExternalSym<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
+  def : TerRMWPatExternalSym<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
+
+  def : TerRMWPatExternalSym<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatExternalSym<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatExternalSym<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatExternalSym<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
+
+  // Truncating-extending ternary RMWs with just a constant offset
+  def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
+  def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
+
+  def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
+
+  def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
+  def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
+
+  def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
+
+  def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
+  def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
+
+  def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
+  def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
+  def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
+  def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
+}
+
+let Predicates = [HasAtomics] in {
+defm : TerRMWTruncExtPattern<
+  atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64,
+  ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32,
+  ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64,
+  ATOMIC_RMW32_U_CMPXCHG_I64>;
+}
+
+//===----------------------------------------------------------------------===//
+// Atomic wait / notify
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 1 in {
+defm ATOMIC_NOTIFY :
+  I<(outs I32:$dst),
+    (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count),
+    (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+    "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count",
+    "atomic.notify \t${off}, ${p2align}", 0xfe00>;
+let mayLoad = 1 in {
+defm ATOMIC_WAIT_I32 :
+  I<(outs I32:$dst),
+    (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout),
+    (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+    "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
+    "i32.atomic.wait \t${off}, ${p2align}", 0xfe01>;
+defm ATOMIC_WAIT_I64 :
+  I<(outs I32:$dst),
+    (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout),
+    (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+    "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
+    "i64.atomic.wait \t${off}, ${p2align}", 0xfe02>;
+} // mayLoad = 1
+} // hasSideEffects = 1
+
+let Predicates = [HasAtomics] in {
+// Select notifys with no constant offset.
+class NotifyPatNoOffset<Intrinsic kind> :
+  Pat<(i32 (kind I32:$addr, I32:$count)),
+      (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>;
+def : NotifyPatNoOffset<int_wasm_atomic_notify>;
+
+// Select notifys with a constant offset.
+
+// Pattern with address + immediate offset
+class NotifyPatImmOff<Intrinsic kind, PatFrag operand> :
+  Pat<(i32 (kind (operand I32:$addr, imm:$off), I32:$count)),
+      (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>;
+def : NotifyPatImmOff<int_wasm_atomic_notify, regPlusImm>;
+def : NotifyPatImmOff<int_wasm_atomic_notify, or_is_add>;
+
+class NotifyPatGlobalAddr<Intrinsic kind> :
+  Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
+                 I32:$count)),
+      (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>;
+def : NotifyPatGlobalAddr<int_wasm_atomic_notify>;
+
+class NotifyPatExternalSym<Intrinsic kind> :
+  Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
+                 I32:$count)),
+      (ATOMIC_NOTIFY 0, texternalsym:$off, I32:$addr, I32:$count)>;
+def : NotifyPatExternalSym<int_wasm_atomic_notify>;
+
+// Select notifys with just a constant offset.
+class NotifyPatOffsetOnly<Intrinsic kind> :
+  Pat<(i32 (kind imm:$off, I32:$count)),
+      (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>;
+def : NotifyPatOffsetOnly<int_wasm_atomic_notify>;
+
+class NotifyPatGlobalAddrOffOnly<Intrinsic kind> :
+  Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), I32:$count)),
+      (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>;
+def : NotifyPatGlobalAddrOffOnly<int_wasm_atomic_notify>;
+
+class NotifyPatExternSymOffOnly<Intrinsic kind> :
+  Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), I32:$count)),
+      (ATOMIC_NOTIFY 0, texternalsym:$off, (CONST_I32 0), I32:$count)>;
+def : NotifyPatExternSymOffOnly<int_wasm_atomic_notify>;
+
+// Select waits with no constant offset.
+class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)),
+      (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+// Select waits with a constant offset.
+
+// Pattern with address + immediate offset
+class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> :
+  Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)),
+      (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>;
+def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>;
+def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>;
+def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>;
+
+class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
+                 ty:$exp, I64:$timeout)),
+      (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+class WaitPatExternalSym<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
+                 ty:$exp, I64:$timeout)),
+      (inst 0, texternalsym:$off, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatExternalSym<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatExternalSym<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset.
+class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
+      (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>;
+def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)),
+      (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>;
+def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+class WaitPatExternSymOffOnly<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), ty:$exp,
+                 I64:$timeout)),
+      (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>;
+def : WaitPatExternSymOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatExternSymOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+} // Predicates = [HasAtomics]
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrCall.td b/lib/Target/WebAssembly/WebAssemblyInstrCall.td
index 34262752430c..07839b790114 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrCall.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrCall.td
@@ -15,8 +15,6 @@
 // TODO: addr64: These currently assume the callee address is 32-bit.
 // FIXME: add $type to first call_indirect asmstr (and maybe $flags)
 
-let Defs = [ARGUMENTS] in {
-
 // Call sequence markers. These have an immediate which represents the amount of
 // stack space to allocate or free, which is used for varargs lowering.
 let Uses = [SP32, SP64], Defs = [SP32, SP64], isCodeGenOnly = 1 in {
@@ -52,34 +50,35 @@ multiclass CALL<WebAssemblyRegClass vt, string prefix> {
 }
 
 multiclass SIMD_CALL<ValueType vt, string prefix> {
-  defm CALL_#vt : SIMD_I<(outs V128:$dst), (ins function32_op:$callee,
-                           variable_ops),
-                         (outs), (ins function32_op:$callee),
-                         [(set (vt V128:$dst),
-                            (WebAssemblycall1 (i32 imm:$callee)))],
-                         !strconcat(prefix, "call\t$dst, $callee"),
-                         !strconcat(prefix, "call\t$callee"),
-                         0x10>;
+
+  defm CALL_#vt : I<(outs V128:$dst), (ins function32_op:$callee, variable_ops),
+                    (outs), (ins function32_op:$callee),
+                    [(set (vt V128:$dst),
+                      (WebAssemblycall1 (i32 imm:$callee)))],
+                    !strconcat(prefix, "call\t$dst, $callee"),
+                    !strconcat(prefix, "call\t$callee"),
+                    0x10>,
+                  Requires<[HasSIMD128]>;
 
   let isCodeGenOnly = 1 in {
-    defm PCALL_INDIRECT_#vt : SIMD_I<(outs V128:$dst),
-                                     (ins I32:$callee, variable_ops),
-                                     (outs), (ins I32:$callee),
-                                     [(set (vt V128:$dst),
-                                           (WebAssemblycall1 I32:$callee))],
-                                     "PSEUDO CALL INDIRECT\t$callee",
-                                     "PSEUDO CALL INDIRECT\t$callee">;
+    defm PCALL_INDIRECT_#vt : I<(outs V128:$dst),
+                                (ins I32:$callee, variable_ops),
+                                (outs), (ins I32:$callee),
+                                [(set (vt V128:$dst),
+                                      (WebAssemblycall1 I32:$callee))],
+                                "PSEUDO CALL INDIRECT\t$callee",
+                                "PSEUDO CALL INDIRECT\t$callee">,
+                              Requires<[HasSIMD128]>;
   } // isCodeGenOnly = 1
 
-  defm CALL_INDIRECT_#vt : SIMD_I<(outs V128:$dst),
-                                  (ins TypeIndex:$type, i32imm:$flags,
-                                        variable_ops),
-                                  (outs), (ins TypeIndex:$type, i32imm:$flags),
-                                  [],
-                                  !strconcat(prefix,
-                                    "call_indirect\t$dst"),
-                                  !strconcat(prefix, "call_indirect\t$type"),
-                                  0x11>;
+  defm CALL_INDIRECT_#vt : I<(outs V128:$dst),
+                             (ins TypeIndex:$type, i32imm:$flags, variable_ops),
+                             (outs), (ins TypeIndex:$type, i32imm:$flags),
+                             [],
+                             !strconcat(prefix, "call_indirect\t$dst"),
+                             !strconcat(prefix, "call_indirect\t$type"),
+                             0x11>,
+                           Requires<[HasSIMD128]>;
 }
 
 let Uses = [SP32, SP64], isCall = 1 in {
@@ -88,10 +87,12 @@ let Uses = [SP32, SP64], isCall = 1 in {
   defm "" : CALL<F32, "f32.">;
   defm "" : CALL<F64, "f64.">;
   defm "" : CALL<EXCEPT_REF, "except_ref.">;
-  defm "" : SIMD_CALL<v16i8, "i8x16.">;
-  defm "" : SIMD_CALL<v8i16, "i16x8.">;
-  defm "" : SIMD_CALL<v4i32, "i32x4.">;
-  defm "" : SIMD_CALL<v4f32, "f32x4.">;
+  defm "" : SIMD_CALL<v16i8, "v128.">;
+  defm "" : SIMD_CALL<v8i16, "v128.">;
+  defm "" : SIMD_CALL<v4i32, "v128.">;
+  defm "" : SIMD_CALL<v2i64, "v128.">;
+  defm "" : SIMD_CALL<v4f32, "v128.">;
+  defm "" : SIMD_CALL<v2f64, "v128.">;
 
   defm CALL_VOID : I<(outs), (ins function32_op:$callee, variable_ops),
                      (outs), (ins function32_op:$callee),
@@ -115,8 +116,6 @@ let Uses = [SP32, SP64], isCall = 1 in {
                               0x11>;
 } // Uses = [SP32,SP64], isCall = 1
 
-} // Defs = [ARGUMENTS]
-
 // Patterns for matching a direct call to a global address.
 def : Pat<(i32 (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
           (CALL_I32 tglobaladdr:$callee)>;
@@ -132,8 +131,12 @@ def : Pat<(v8i16 (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
           (CALL_v8i16 tglobaladdr:$callee)>, Requires<[HasSIMD128]>;
 def : Pat<(v4i32 (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
           (CALL_v4i32 tglobaladdr:$callee)>, Requires<[HasSIMD128]>;
+def : Pat<(v2i64 (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
+          (CALL_v2i64 tglobaladdr:$callee)>, Requires<[HasSIMD128]>;
 def : Pat<(v4f32 (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
           (CALL_v4f32 tglobaladdr:$callee)>, Requires<[HasSIMD128]>;
+def : Pat<(v2f64 (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
+          (CALL_v2f64 tglobaladdr:$callee)>, Requires<[HasSIMD128]>;
 def : Pat<(ExceptRef
            (WebAssemblycall1 (WebAssemblywrapper tglobaladdr:$callee))),
           (CALL_EXCEPT_REF tglobaladdr:$callee)>;
@@ -155,8 +158,12 @@ def : Pat<(v8i16 (WebAssemblycall1 (WebAssemblywrapper texternalsym:$callee))),
           (CALL_v8i16 texternalsym:$callee)>, Requires<[HasSIMD128]>;
 def : Pat<(v4i32 (WebAssemblycall1 (WebAssemblywrapper texternalsym:$callee))),
           (CALL_v4i32 texternalsym:$callee)>, Requires<[HasSIMD128]>;
+def : Pat<(v2i64 (WebAssemblycall1 (WebAssemblywrapper texternalsym:$callee))),
+          (CALL_v2i64 texternalsym:$callee)>, Requires<[HasSIMD128]>;
 def : Pat<(v4f32 (WebAssemblycall1 (WebAssemblywrapper texternalsym:$callee))),
           (CALL_v4f32 texternalsym:$callee)>, Requires<[HasSIMD128]>;
+def : Pat<(v2f64 (WebAssemblycall1 (WebAssemblywrapper texternalsym:$callee))),
+          (CALL_v2f64 texternalsym:$callee)>, Requires<[HasSIMD128]>;
 def : Pat<(ExceptRef
            (WebAssemblycall1 (WebAssemblywrapper texternalsym:$callee))),
           (CALL_EXCEPT_REF texternalsym:$callee)>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrControl.td b/lib/Target/WebAssembly/WebAssemblyInstrControl.td
index d90244b90662..7eb6cbf4d249 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrControl.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrControl.td
@@ -12,8 +12,6 @@
 ///
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
-
 let isBranch = 1, isTerminator = 1, hasCtrlDep = 1 in {
 // The condition operand is a boolean value which WebAssembly represents as i32.
 defm BR_IF : I<(outs), (ins bb_op:$dst, I32:$cond),
@@ -30,47 +28,37 @@ defm BR   : NRI<(outs), (ins bb_op:$dst),
 } // isBarrier = 1
 } // isBranch = 1, isTerminator = 1, hasCtrlDep = 1
 
-} // Defs = [ARGUMENTS]
-
 def : Pat<(brcond (i32 (setne I32:$cond, 0)), bb:$dst),
           (BR_IF bb_op:$dst, I32:$cond)>;
 def : Pat<(brcond (i32 (seteq I32:$cond, 0)), bb:$dst),
           (BR_UNLESS bb_op:$dst, I32:$cond)>;
 
-let Defs = [ARGUMENTS] in {
+// A list of branch targets enclosed in {} and separated by comma.
+// Used by br_table only.
+def BrListAsmOperand : AsmOperandClass { let Name = "BrList"; }
+let OperandNamespace = "WebAssembly" in {
+let OperandType = "OPERAND_BRLIST" in {
+def brlist : Operand<i32> {
+  let ParserMatchClass = BrListAsmOperand;
+  let PrintMethod = "printBrList";
+}
+} // OPERAND_BRLIST
+} // OperandNamespace = "WebAssembly"
 
 // TODO: SelectionDAG's lowering insists on using a pointer as the index for
 // jump tables, so in practice we don't ever use BR_TABLE_I64 in wasm32 mode
 // currently.
-// Set TSFlags{0} to 1 to indicate that the variable_ops are immediates.
-// Set TSFlags{1} to 1 to indicate that the immediates represent labels.
-// FIXME: this can't inherit from I<> since there is no way to inherit from a
-// multiclass and still have the let statements.
 let isTerminator = 1, hasCtrlDep = 1, isBarrier = 1 in {
-def BR_TABLE_I32 : NI<(outs), (ins I32:$index, variable_ops),
-                      [(WebAssemblybr_table I32:$index)], 0,
-                      "br_table \t$index", 0x0e> {
-  let TSFlags{0} = 1;
-  let TSFlags{1} = 1;
-}
-def BR_TABLE_I32_S : NI<(outs), (ins I32:$index),
-                        [], 1,
-                        "br_table \t$index", 0x0e> {
-  let TSFlags{0} = 1;
-  let TSFlags{1} = 1;
-}
-def BR_TABLE_I64 : NI<(outs), (ins I64:$index, variable_ops),
-                      [(WebAssemblybr_table I64:$index)], 0,
-                      "br_table \t$index"> {
-  let TSFlags{0} = 1;
-  let TSFlags{1} = 1;
-}
-def BR_TABLE_I64_S : NI<(outs), (ins I64:$index),
-                        [], 1,
-                        "br_table \t$index"> {
-  let TSFlags{0} = 1;
-  let TSFlags{1} = 1;
-}
+defm BR_TABLE_I32 : I<(outs), (ins I32:$index, variable_ops),
+                      (outs), (ins brlist:$brl),
+                      [(WebAssemblybr_table I32:$index)],
+                      "br_table \t$index", "br_table \t$brl",
+                      0x0e>;
+defm BR_TABLE_I64 : I<(outs), (ins I64:$index, variable_ops),
+                      (outs), (ins brlist:$brl),
+                      [(WebAssemblybr_table I64:$index)],
+                      "br_table \t$index", "br_table \t$brl",
+                      0x0e>;
 } // isTerminator = 1, hasCtrlDep = 1, isBarrier = 1
 
 // This is technically a control-flow instruction, since all it affects is the
@@ -81,13 +69,19 @@ defm NOP : NRI<(outs), (ins), [], "nop", 0x01>;
 // These use/clobber VALUE_STACK to prevent them from being moved into the
 // middle of an expression tree.
 let Uses = [VALUE_STACK], Defs = [VALUE_STACK] in {
-defm BLOCK     : NRI<(outs), (ins Signature:$sig), [], "block   \t$sig", 0x02>;
-defm LOOP      : NRI<(outs), (ins Signature:$sig), [], "loop    \t$sig", 0x03>;
+defm BLOCK : NRI<(outs), (ins Signature:$sig), [], "block   \t$sig", 0x02>;
+defm LOOP  : NRI<(outs), (ins Signature:$sig), [], "loop    \t$sig", 0x03>;
+
+defm IF : I<(outs), (ins Signature:$sig, I32:$cond),
+            (outs), (ins Signature:$sig),
+            [], "if    \t$sig, $cond", "if    \t$sig", 0x04>;
+defm ELSE : NRI<(outs), (ins), [], "else", 0x05>;
 
-// END_BLOCK, END_LOOP, and END_FUNCTION are represented with the same opcode in
-// wasm.
+// END_BLOCK, END_LOOP, END_IF and END_FUNCTION are represented with the same
+// opcode in wasm.
 defm END_BLOCK : NRI<(outs), (ins), [], "end_block", 0x0b>;
 defm END_LOOP  : NRI<(outs), (ins), [], "end_loop", 0x0b>;
+defm END_IF    : NRI<(outs), (ins), [], "end_if", 0x0b>;
 let isTerminator = 1, isBarrier = 1 in
 defm END_FUNCTION : NRI<(outs), (ins), [], "end_function", 0x0b>;
 } // Uses = [VALUE_STACK], Defs = [VALUE_STACK]
@@ -103,14 +97,16 @@ multiclass RETURN<WebAssemblyRegClass vt> {
 }
 
 multiclass SIMD_RETURN<ValueType vt> {
-  defm RETURN_#vt : SIMD_I<(outs), (ins V128:$val), (outs), (ins),
-                           [(WebAssemblyreturn (vt V128:$val))],
-                           "return  \t$val", "return", 0x0f>;
+  defm RETURN_#vt : I<(outs), (ins V128:$val), (outs), (ins),
+                      [(WebAssemblyreturn (vt V128:$val))],
+                      "return  \t$val", "return", 0x0f>,
+                    Requires<[HasSIMD128]>;
   // Equivalent to RETURN_#vt, for use at the end of a function when wasm
   // semantics return by falling off the end of the block.
   let isCodeGenOnly = 1 in
-  defm FALLTHROUGH_RETURN_#vt : SIMD_I<(outs), (ins V128:$val), (outs), (ins),
-                                       []>;
+  defm FALLTHROUGH_RETURN_#vt : I<(outs), (ins V128:$val), (outs), (ins),
+                                  []>,
+                                Requires<[HasSIMD128]>;
 }
 
 let isTerminator = 1, hasCtrlDep = 1, isBarrier = 1 in {
@@ -124,7 +120,9 @@ let isReturn = 1 in {
   defm "": SIMD_RETURN<v16i8>;
   defm "": SIMD_RETURN<v8i16>;
   defm "": SIMD_RETURN<v4i32>;
+  defm "": SIMD_RETURN<v2i64>;
   defm "": SIMD_RETURN<v4f32>;
+  defm "": SIMD_RETURN<v2f64>;
 
   defm RETURN_VOID : NRI<(outs), (ins), [(WebAssemblyreturn)], "return", 0x0f>;
 
@@ -144,14 +142,16 @@ let Predicates = [HasExceptionHandling] in {
 
 // Throwing an exception: throw / rethrow
 let isTerminator = 1, hasCtrlDep = 1, isBarrier = 1 in {
-defm THROW_I32 : I<(outs), (ins i32imm:$tag, I32:$val),
-                   (outs), (ins i32imm:$tag),
-                   [(int_wasm_throw imm:$tag, I32:$val)],
+defm THROW_I32 : I<(outs), (ins event_op:$tag, I32:$val),
+                   (outs), (ins event_op:$tag),
+                   [(WebAssemblythrow (WebAssemblywrapper texternalsym:$tag),
+                                      I32:$val)],
                    "throw   \t$tag, $val", "throw   \t$tag",
                    0x08>;
-defm THROW_I64 : I<(outs), (ins i32imm:$tag, I64:$val),
-                   (outs), (ins i32imm:$tag),
-                   [(int_wasm_throw imm:$tag, I64:$val)],
+defm THROW_I64 : I<(outs), (ins event_op:$tag, I64:$val),
+                   (outs), (ins event_op:$tag),
+                   [(WebAssemblythrow (WebAssemblywrapper texternalsym:$tag),
+                                      I64:$val)],
                    "throw   \t$tag, $val", "throw   \t$tag",
                    0x08>;
 defm RETHROW : NRI<(outs), (ins bb_op:$dst), [], "rethrow \t$dst", 0x09>;
@@ -168,7 +168,7 @@ defm END_TRY : NRI<(outs), (ins), [], "end_try", 0x0b>;
 } // Uses = [VALUE_STACK], Defs = [VALUE_STACK]
 
 // Catching an exception: catch / catch_all
-let hasCtrlDep = 1 in {
+let hasCtrlDep = 1, hasSideEffects = 1 in {
 defm CATCH_I32 : I<(outs I32:$dst), (ins i32imm:$tag),
                    (outs), (ins i32imm:$tag),
                    [(set I32:$dst, (int_wasm_catch imm:$tag))],
@@ -181,14 +181,10 @@ defm CATCH_ALL : NRI<(outs), (ins), [], "catch_all", 0x05>;
 }
 
 // Pseudo instructions: cleanupret / catchret
-// They are not return instructions in wasm, but setting 'isReturn' to true as
-// in X86 is necessary for computing EH scope membership.
 let isTerminator = 1, hasSideEffects = 1, isBarrier = 1, hasCtrlDep = 1,
-    isCodeGenOnly = 1, isReturn = 1 in {
+    isCodeGenOnly = 1, isEHScopeReturn = 1 in {
   defm CLEANUPRET : NRI<(outs), (ins), [(cleanupret)], "", 0>;
   defm CATCHRET : NRI<(outs), (ins bb_op:$dst, bb_op:$from),
                    [(catchret bb:$dst, bb:$from)], "", 0>;
 }
 }
-
-} // Defs = [ARGUMENTS]
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrConv.td b/lib/Target/WebAssembly/WebAssemblyInstrConv.td
index c89c1b549816..e128656a142c 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrConv.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrConv.td
@@ -13,19 +13,17 @@
 ///
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
-
 defm I32_WRAP_I64 : I<(outs I32:$dst), (ins I64:$src), (outs), (ins),
                       [(set I32:$dst, (trunc I64:$src))],
-                      "i32.wrap/i64\t$dst, $src", "i32.wrap/i64", 0xa7>;
+                      "i32.wrap_i64\t$dst, $src", "i32.wrap_i64", 0xa7>;
 
 defm I64_EXTEND_S_I32 : I<(outs I64:$dst), (ins I32:$src), (outs), (ins),
                           [(set I64:$dst, (sext I32:$src))],
-                          "i64.extend_s/i32\t$dst, $src", "i64.extend_s/i32",
+                          "i64.extend_i32_s\t$dst, $src", "i64.extend_i32_s",
                           0xac>;
 defm I64_EXTEND_U_I32 : I<(outs I64:$dst), (ins I32:$src), (outs), (ins),
                           [(set I64:$dst, (zext I32:$src))],
-                          "i64.extend_u/i32\t$dst, $src", "i64.extend_u/i32",
+                          "i64.extend_i32_u\t$dst, $src", "i64.extend_i32_u",
                           0xad>;
 
 let Predicates = [HasSignExt] in {
@@ -51,58 +49,72 @@ defm I64_EXTEND32_S_I64 : I<(outs I64:$dst), (ins I64:$src), (outs), (ins),
                             0xc4>;
 } // Predicates = [HasSignExt]
 
-} // defs = [ARGUMENTS]
-
 // Expand a "don't care" extend into zero-extend (chosen over sign-extend
 // somewhat arbitrarily, although it favors popular hardware architectures
 // and is conceptually a simpler operation).
 def : Pat<(i64 (anyext I32:$src)), (I64_EXTEND_U_I32 I32:$src)>;
 
-let Defs = [ARGUMENTS] in {
-
 // Conversion from floating point to integer instructions which don't trap on
 // overflow or invalid.
 defm I32_TRUNC_S_SAT_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_sint F32:$src))],
-                             "i32.trunc_s:sat/f32\t$dst, $src",
-                             "i32.trunc_s:sat/f32", 0xfc00>,
+                             "i32.trunc_sat_f32_s\t$dst, $src",
+                             "i32.trunc_sat_f32_s", 0xfc00>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I32_TRUNC_U_SAT_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_uint F32:$src))],
-                             "i32.trunc_u:sat/f32\t$dst, $src",
-                             "i32.trunc_u:sat/f32", 0xfc01>,
+                             "i32.trunc_sat_f32_u\t$dst, $src",
+                             "i32.trunc_sat_f32_u", 0xfc01>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_S_SAT_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_sint F32:$src))],
-                             "i64.trunc_s:sat/f32\t$dst, $src",
-                             "i64.trunc_s:sat/f32", 0xfc04>,
+                             "i64.trunc_sat_f32_s\t$dst, $src",
+                             "i64.trunc_sat_f32_s", 0xfc04>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_U_SAT_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_uint F32:$src))],
-                             "i64.trunc_u:sat/f32\t$dst, $src",
-                             "i64.trunc_u:sat/f32", 0xfc05>,
+                             "i64.trunc_sat_f32_u\t$dst, $src",
+                             "i64.trunc_sat_f32_u", 0xfc05>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I32_TRUNC_S_SAT_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_sint F64:$src))],
-                             "i32.trunc_s:sat/f64\t$dst, $src",
-                             "i32.trunc_s:sat/f64", 0xfc02>,
+                             "i32.trunc_sat_f64_s\t$dst, $src",
+                             "i32.trunc_sat_f64_s", 0xfc02>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I32_TRUNC_U_SAT_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_uint F64:$src))],
-                             "i32.trunc_u:sat/f64\t$dst, $src",
-                             "i32.trunc_u:sat/f64", 0xfc03>,
+                             "i32.trunc_sat_f64_u\t$dst, $src",
+                             "i32.trunc_sat_f64_u", 0xfc03>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_S_SAT_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_sint F64:$src))],
-                             "i64.trunc_s:sat/f64\t$dst, $src",
-                             "i64.trunc_s:sat/f64", 0xfc06>,
+                             "i64.trunc_sat_f64_s\t$dst, $src",
+                             "i64.trunc_sat_f64_s", 0xfc06>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_U_SAT_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_uint F64:$src))],
-                             "i64.trunc_u:sat/f64\t$dst, $src",
-                             "i64.trunc_u:sat/f64", 0xfc07>,
+                             "i64.trunc_sat_f64_u\t$dst, $src",
+                             "i64.trunc_sat_f64_u", 0xfc07>,
                              Requires<[HasNontrappingFPToInt]>;
 
+// Lower llvm.wasm.trunc.saturate.* to saturating instructions
+def : Pat<(int_wasm_trunc_saturate_signed F32:$src),
+          (I32_TRUNC_S_SAT_F32 F32:$src)>;
+def : Pat<(int_wasm_trunc_saturate_unsigned F32:$src),
+          (I32_TRUNC_U_SAT_F32 F32:$src)>;
+def : Pat<(int_wasm_trunc_saturate_signed F64:$src),
+          (I32_TRUNC_S_SAT_F64 F64:$src)>;
+def : Pat<(int_wasm_trunc_saturate_unsigned F64:$src),
+          (I32_TRUNC_U_SAT_F64 F64:$src)>;
+def : Pat<(int_wasm_trunc_saturate_signed F32:$src),
+          (I64_TRUNC_S_SAT_F32 F32:$src)>;
+def : Pat<(int_wasm_trunc_saturate_unsigned F32:$src),
+          (I64_TRUNC_U_SAT_F32 F32:$src)>;
+def : Pat<(int_wasm_trunc_saturate_signed F64:$src),
+          (I64_TRUNC_S_SAT_F64 F64:$src)>;
+def : Pat<(int_wasm_trunc_saturate_unsigned F64:$src),
+          (I64_TRUNC_U_SAT_F64 F64:$src)>;
+
 // Conversion from floating point to integer pseudo-instructions which don't
 // trap on overflow or invalid.
 let usesCustomInserter = 1, isCodeGenOnly = 1 in {
@@ -135,88 +147,86 @@ defm FP_TO_UINT_I64_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
 // Conversion from floating point to integer traps on overflow and invalid.
 let hasSideEffects = 1 in {
 defm I32_TRUNC_S_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i32.trunc_s/f32\t$dst, $src", "i32.trunc_s/f32",
+                         [], "i32.trunc_f32_s\t$dst, $src", "i32.trunc_f32_s",
                          0xa8>;
 defm I32_TRUNC_U_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i32.trunc_u/f32\t$dst, $src", "i32.trunc_u/f32",
+                         [], "i32.trunc_f32_u\t$dst, $src", "i32.trunc_f32_u",
                          0xa9>;
 defm I64_TRUNC_S_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i64.trunc_s/f32\t$dst, $src", "i64.trunc_s/f32",
+                         [], "i64.trunc_f32_s\t$dst, $src", "i64.trunc_f32_s",
                          0xae>;
 defm I64_TRUNC_U_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i64.trunc_u/f32\t$dst, $src", "i64.trunc_u/f32",
+                         [], "i64.trunc_f32_u\t$dst, $src", "i64.trunc_f32_u",
                          0xaf>;
 defm I32_TRUNC_S_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i32.trunc_s/f64\t$dst, $src", "i32.trunc_s/f64",
+                         [], "i32.trunc_f64_s\t$dst, $src", "i32.trunc_f64_s",
                          0xaa>;
 defm I32_TRUNC_U_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i32.trunc_u/f64\t$dst, $src", "i32.trunc_u/f64",
+                         [], "i32.trunc_f64_u\t$dst, $src", "i32.trunc_f64_u",
                          0xab>;
 defm I64_TRUNC_S_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i64.trunc_s/f64\t$dst, $src", "i64.trunc_s/f64",
+                         [], "i64.trunc_f64_s\t$dst, $src", "i64.trunc_f64_s",
                          0xb0>;
 defm I64_TRUNC_U_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i64.trunc_u/f64\t$dst, $src", "i64.trunc_u/f64",
+                         [], "i64.trunc_f64_u\t$dst, $src", "i64.trunc_f64_u",
                          0xb1>;
 } // hasSideEffects = 1
 
 defm F32_CONVERT_S_I32 : I<(outs F32:$dst), (ins I32:$src), (outs), (ins),
                            [(set F32:$dst, (sint_to_fp I32:$src))],
-                           "f32.convert_s/i32\t$dst, $src", "f32.convert_s/i32",
+                           "f32.convert_i32_s\t$dst, $src", "f32.convert_i32_s",
                            0xb2>;
 defm F32_CONVERT_U_I32 : I<(outs F32:$dst), (ins I32:$src), (outs), (ins),
                            [(set F32:$dst, (uint_to_fp I32:$src))],
-                           "f32.convert_u/i32\t$dst, $src", "f32.convert_u/i32",
+                           "f32.convert_i32_u\t$dst, $src", "f32.convert_i32_u",
                            0xb3>;
 defm F64_CONVERT_S_I32 : I<(outs F64:$dst), (ins I32:$src), (outs), (ins),
                            [(set F64:$dst, (sint_to_fp I32:$src))],
-                           "f64.convert_s/i32\t$dst, $src", "f64.convert_s/i32",
+                           "f64.convert_i32_s\t$dst, $src", "f64.convert_i32_s",
                            0xb7>;
 defm F64_CONVERT_U_I32 : I<(outs F64:$dst), (ins I32:$src), (outs), (ins),
                            [(set F64:$dst, (uint_to_fp I32:$src))],
-                           "f64.convert_u/i32\t$dst, $src", "f64.convert_u/i32",
+                           "f64.convert_i32_u\t$dst, $src", "f64.convert_i32_u",
                            0xb8>;
 defm F32_CONVERT_S_I64 : I<(outs F32:$dst), (ins I64:$src), (outs), (ins),
                            [(set F32:$dst, (sint_to_fp I64:$src))],
-                           "f32.convert_s/i64\t$dst, $src", "f32.convert_s/i64",
+                           "f32.convert_i64_s\t$dst, $src", "f32.convert_i64_s",
                            0xb4>;
 defm F32_CONVERT_U_I64 : I<(outs F32:$dst), (ins I64:$src), (outs), (ins),
                            [(set F32:$dst, (uint_to_fp I64:$src))],
-                           "f32.convert_u/i64\t$dst, $src", "f32.convert_u/i64",
+                           "f32.convert_i64_u\t$dst, $src", "f32.convert_i64_u",
                            0xb5>;
 defm F64_CONVERT_S_I64 : I<(outs F64:$dst), (ins I64:$src), (outs), (ins),
                            [(set F64:$dst, (sint_to_fp I64:$src))],
-                           "f64.convert_s/i64\t$dst, $src", "f64.convert_s/i64",
+                           "f64.convert_i64_s\t$dst, $src", "f64.convert_i64_s",
                            0xb9>;
 defm F64_CONVERT_U_I64 : I<(outs F64:$dst), (ins I64:$src), (outs), (ins),
                            [(set F64:$dst, (uint_to_fp I64:$src))],
-                           "f64.convert_u/i64\t$dst, $src", "f64.convert_u/i64",
+                           "f64.convert_i64_u\t$dst, $src", "f64.convert_i64_u",
                            0xba>;
 
 defm F64_PROMOTE_F32 : I<(outs F64:$dst), (ins F32:$src), (outs), (ins),
                          [(set F64:$dst, (fpextend F32:$src))],
-                         "f64.promote/f32\t$dst, $src", "f64.promote/f32",
+                         "f64.promote_f32\t$dst, $src", "f64.promote_f32",
                          0xbb>;
 defm F32_DEMOTE_F64 : I<(outs F32:$dst), (ins F64:$src), (outs), (ins),
                         [(set F32:$dst, (fpround F64:$src))],
-                        "f32.demote/f64\t$dst, $src", "f32.demote/f64",
+                        "f32.demote_f64\t$dst, $src", "f32.demote_f64",
                         0xb6>;
 
 defm I32_REINTERPRET_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                              [(set I32:$dst, (bitconvert F32:$src))],
-                             "i32.reinterpret/f32\t$dst, $src",
-                             "i32.reinterpret/f32", 0xbc>;
+                             "i32.reinterpret_f32\t$dst, $src",
+                             "i32.reinterpret_f32", 0xbc>;
 defm F32_REINTERPRET_I32 : I<(outs F32:$dst), (ins I32:$src), (outs), (ins),
                              [(set F32:$dst, (bitconvert I32:$src))],
-                             "f32.reinterpret/i32\t$dst, $src",
-                             "f32.reinterpret/i32", 0xbe>;
+                             "f32.reinterpret_i32\t$dst, $src",
+                             "f32.reinterpret_i32", 0xbe>;
 defm I64_REINTERPRET_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (bitconvert F64:$src))],
-                             "i64.reinterpret/f64\t$dst, $src",
-                             "i64.reinterpret/f64", 0xbd>;
+                             "i64.reinterpret_f64\t$dst, $src",
+                             "i64.reinterpret_f64", 0xbd>;
 defm F64_REINTERPRET_I64 : I<(outs F64:$dst), (ins I64:$src), (outs), (ins),
                              [(set F64:$dst, (bitconvert I64:$src))],
-                             "f64.reinterpret/i64\t$dst, $src",
-                             "f64.reinterpret/i64", 0xbf>;
-
-} // Defs = [ARGUMENTS]
+                             "f64.reinterpret_i64\t$dst, $src",
+                             "f64.reinterpret_i64", 0xbf>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td b/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td
index 41b39f69e51c..a251d60b89ee 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td
@@ -12,8 +12,6 @@
 ///
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
-
 defm SELECT_EXCEPT_REF : I<(outs EXCEPT_REF:$dst),
                            (ins EXCEPT_REF:$lhs, EXCEPT_REF:$rhs, I32:$cond),
                            (outs), (ins),
@@ -23,8 +21,6 @@ defm SELECT_EXCEPT_REF : I<(outs EXCEPT_REF:$dst),
                            "except_ref.select\t$dst, $lhs, $rhs, $cond",
                            "except_ref.select", 0x1b>;
 
-} // Defs = [ARGUMENTS]
-
 def : Pat<(select (i32 (setne I32:$cond, 0)), EXCEPT_REF:$lhs, EXCEPT_REF:$rhs),
           (SELECT_EXCEPT_REF EXCEPT_REF:$lhs, EXCEPT_REF:$rhs, I32:$cond)>;
 def : Pat<(select (i32 (seteq I32:$cond, 0)), EXCEPT_REF:$lhs, EXCEPT_REF:$rhs),
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrFloat.td b/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
index 8db75d38942b..c5290f00b431 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
@@ -12,7 +12,38 @@
 ///
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
+multiclass UnaryFP<SDNode node, string name, bits<32> f32Inst,
+                   bits<32> f64Inst> {
+  defm _F32 : I<(outs F32:$dst), (ins F32:$src), (outs), (ins),
+                [(set F32:$dst, (node F32:$src))],
+                !strconcat("f32.", !strconcat(name, "\t$dst, $src")),
+                !strconcat("f32.", name), f32Inst>;
+  defm _F64 : I<(outs F64:$dst), (ins F64:$src), (outs), (ins),
+                [(set F64:$dst, (node F64:$src))],
+                !strconcat("f64.", !strconcat(name, "\t$dst, $src")),
+                !strconcat("f64.", name), f64Inst>;
+}
+multiclass BinaryFP<SDNode node, string name, bits<32> f32Inst,
+                    bits<32> f64Inst> {
+  defm _F32 : I<(outs F32:$dst), (ins F32:$lhs, F32:$rhs), (outs), (ins),
+                [(set F32:$dst, (node F32:$lhs, F32:$rhs))],
+                !strconcat("f32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("f32.", name), f32Inst>;
+  defm _F64 : I<(outs F64:$dst), (ins F64:$lhs, F64:$rhs), (outs), (ins),
+                [(set F64:$dst, (node F64:$lhs, F64:$rhs))],
+                !strconcat("f64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("f64.", name), f64Inst>;
+}
+multiclass ComparisonFP<CondCode cond, string name, bits<32> f32Inst, bits<32> f64Inst> {
+  defm _F32 : I<(outs I32:$dst), (ins F32:$lhs, F32:$rhs), (outs), (ins),
+                [(set I32:$dst, (setcc F32:$lhs, F32:$rhs, cond))],
+                !strconcat("f32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("f32.", name), f32Inst>;
+  defm  _F64 : I<(outs I32:$dst), (ins F64:$lhs, F64:$rhs), (outs), (ins),
+                [(set I32:$dst, (setcc F64:$lhs, F64:$rhs, cond))],
+                !strconcat("f64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("f64.", name), f64Inst>;
+}
 
 let isCommutable = 1 in
 defm ADD : BinaryFP<fadd, "add ", 0x92, 0xa0>;
@@ -27,8 +58,8 @@ defm NEG : UnaryFP<fneg, "neg ", 0x8c, 0x9a>;
 defm COPYSIGN : BinaryFP<fcopysign, "copysign", 0x98, 0xa6>;
 
 let isCommutable = 1 in {
-defm MIN : BinaryFP<fminnan, "min ", 0x96, 0xa4>;
-defm MAX : BinaryFP<fmaxnan, "max ", 0x97, 0xa5>;
+defm MIN : BinaryFP<fminimum, "min ", 0x96, 0xa4>;
+defm MAX : BinaryFP<fmaximum, "max ", 0x97, 0xa5>;
 } // isCommutable = 1
 
 defm CEIL : UnaryFP<fceil, "ceil", 0x8d, 0x9b>;
@@ -36,8 +67,6 @@ defm FLOOR : UnaryFP<ffloor, "floor", 0x8e, 0x9c>;
 defm TRUNC : UnaryFP<ftrunc, "trunc", 0x8f, 0x9d>;
 defm NEAREST : UnaryFP<fnearbyint, "nearest", 0x90, 0x9e>;
 
-} // Defs = [ARGUMENTS]
-
 // DAGCombine oddly folds casts into the rhs of copysign. Unfold them.
 def : Pat<(fcopysign F64:$lhs, F32:$rhs),
           (COPYSIGN_F64 F64:$lhs, (F64_PROMOTE_F32 F32:$rhs))>;
@@ -48,8 +77,6 @@ def : Pat<(fcopysign F32:$lhs, F64:$rhs),
 def : Pat<(frint f32:$src), (NEAREST_F32 f32:$src)>;
 def : Pat<(frint f64:$src), (NEAREST_F64 f64:$src)>;
 
-let Defs = [ARGUMENTS] in {
-
 let isCommutable = 1 in {
 defm EQ : ComparisonFP<SETOEQ, "eq  ", 0x5b, 0x61>;
 defm NE : ComparisonFP<SETUNE, "ne  ", 0x5c, 0x62>;
@@ -59,8 +86,6 @@ defm LE : ComparisonFP<SETOLE, "le  ", 0x5f, 0x65>;
 defm GT : ComparisonFP<SETOGT, "gt  ", 0x5e, 0x64>;
 defm GE : ComparisonFP<SETOGE, "ge  ", 0x60, 0x66>;
 
-} // Defs = [ARGUMENTS]
-
 // Don't care floating-point comparisons, supported via other comparisons.
 def : Pat<(seteq f32:$lhs, f32:$rhs), (EQ_F32 f32:$lhs, f32:$rhs)>;
 def : Pat<(setne f32:$lhs, f32:$rhs), (NE_F32 f32:$lhs, f32:$rhs)>;
@@ -75,8 +100,6 @@ def : Pat<(setle f64:$lhs, f64:$rhs), (LE_F64 f64:$lhs, f64:$rhs)>;
 def : Pat<(setgt f64:$lhs, f64:$rhs), (GT_F64 f64:$lhs, f64:$rhs)>;
 def : Pat<(setge f64:$lhs, f64:$rhs), (GE_F64 f64:$lhs, f64:$rhs)>;
 
-let Defs = [ARGUMENTS] in {
-
 defm SELECT_F32 : I<(outs F32:$dst), (ins F32:$lhs, F32:$rhs, I32:$cond),
                     (outs), (ins),
                     [(set F32:$dst, (select I32:$cond, F32:$lhs, F32:$rhs))],
@@ -86,8 +109,6 @@ defm SELECT_F64 : I<(outs F64:$dst), (ins F64:$lhs, F64:$rhs, I32:$cond),
                     [(set F64:$dst, (select I32:$cond, F64:$lhs, F64:$rhs))],
                     "f64.select\t$dst, $lhs, $rhs, $cond", "f64.select", 0x1b>;
 
-} // Defs = [ARGUMENTS]
-
 // ISD::SELECT requires its operand to conform to getBooleanContents, but
 // WebAssembly's select interprets any non-zero value as true, so we can fold
 // a setne with 0 into a select.
@@ -101,3 +122,10 @@ def : Pat<(select (i32 (seteq I32:$cond, 0)), F32:$lhs, F32:$rhs),
           (SELECT_F32 F32:$rhs, F32:$lhs, I32:$cond)>;
 def : Pat<(select (i32 (seteq I32:$cond, 0)), F64:$lhs, F64:$rhs),
           (SELECT_F64 F64:$rhs, F64:$lhs, I32:$cond)>;
+
+// The legalizer inserts an unnecessary `and 1` to make input conform
+// to getBooleanContents, which we can lower away.
+def : Pat<(select (i32 (and I32:$cond, 1)), F32:$lhs, F32:$rhs),
+          (SELECT_F32 F32:$lhs, F32:$rhs, I32:$cond)>;
+def : Pat<(select (i32 (and I32:$cond, 1)), F64:$lhs, F64:$rhs),
+          (SELECT_F64 F64:$lhs, F64:$rhs, I32:$cond)>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrFormats.td b/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
index 403152c80660..15a9714a55a1 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
@@ -15,21 +15,24 @@
 // WebAssembly Instruction Format.
 // We instantiate 2 of these for every actual instruction (register based
 // and stack based), see below.
-class WebAssemblyInst<bits<32> inst, string asmstr, bit stack> : Instruction {
-  field bits<32> Inst = inst; // Instruction encoding.
-  field bit StackBased = stack;
+class WebAssemblyInst<bits<32> inst, string asmstr, string stack> : StackRel,
+  Instruction {
+  bits<32> Inst = inst; // Instruction encoding.
+  string StackBased = stack;
+  string BaseName = NAME;
   let Namespace   = "WebAssembly";
   let Pattern     = [];
   let AsmString   = asmstr;
 }
 
 // Normal instructions. Default instantiation of a WebAssemblyInst.
-class NI<dag oops, dag iops, list<dag> pattern, bit stack, string asmstr = "",
-         bits<32> inst = -1>
+class NI<dag oops, dag iops, list<dag> pattern, string stack,
+         string asmstr = "", bits<32> inst = -1>
     : WebAssemblyInst<inst, asmstr, stack> {
   dag OutOperandList = oops;
   dag InOperandList  = iops;
   let Pattern        = pattern;
+  let Defs           = [ARGUMENTS];
 }
 
 // Generates both register and stack based versions of one actual instruction.
@@ -37,10 +40,10 @@ class NI<dag oops, dag iops, list<dag> pattern, bit stack, string asmstr = "",
 // based version of this instruction, as well as the corresponding asmstr.
 // The register versions have virtual-register operands which correspond to wasm
 // locals or stack locations. Each use and def of the register corresponds to an
-// implicit get_local / set_local or access of stack operands in wasm. These
+// implicit local.get / local.set or access of stack operands in wasm. These
 // instructions are used for ISel and all MI passes. The stack versions of the
 // instructions do not have register operands (they implicitly operate on the
-// stack), and get_locals and set_locals are explicit. The register instructions
+// stack), and local.gets and local.sets are explicit. The register instructions
 // are converted to their corresponding stack instructions before lowering to
 // MC.
 // Every instruction should want to be based on this multi-class to guarantee
@@ -48,8 +51,10 @@ class NI<dag oops, dag iops, list<dag> pattern, bit stack, string asmstr = "",
 multiclass I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
              list<dag> pattern_r, string asmstr_r = "", string asmstr_s = "",
              bits<32> inst = -1> {
-  def "" : NI<oops_r, iops_r, pattern_r, 0, asmstr_r, inst>;
-  def _S : NI<oops_s, iops_s, [], 1, asmstr_s, inst>;
+  let isCodeGenOnly = 1 in
+  def "" : NI<oops_r, iops_r, pattern_r, "false", asmstr_r, inst>;
+  let BaseName = NAME in
+  def _S : NI<oops_s, iops_s, [], "true", asmstr_s, inst>;
 }
 
 // For instructions that have no register ops, so both sets are the same.
@@ -57,111 +62,3 @@ multiclass NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "",
                bits<32> inst = -1> {
   defm "": I<oops, iops, oops, iops, pattern, asmstr, asmstr, inst>;
 }
-
-multiclass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
-                  list<dag> pattern_r, string asmstr_r = "",
-                  string asmstr_s = "", bits<32> inst = -1> {
-  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
-              inst>,
-            Requires<[HasSIMD128]>;
-}
-
-multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
-                    list<dag> pattern_r, string asmstr_r = "",
-                    string asmstr_s = "", bits<32> inst = -1> {
-  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
-              inst>,
-            Requires<[HasAtomics]>;
-}
-
-// Unary and binary instructions, for the local types that WebAssembly supports.
-multiclass UnaryInt<SDNode node, string name, bits<32> i32Inst,
-                    bits<32> i64Inst> {
-  defm _I32 : I<(outs I32:$dst), (ins I32:$src), (outs), (ins),
-                [(set I32:$dst, (node I32:$src))],
-                !strconcat("i32.", !strconcat(name, "\t$dst, $src")),
-                !strconcat("i32.", name), i32Inst>;
-  defm _I64 : I<(outs I64:$dst), (ins I64:$src), (outs), (ins),
-                [(set I64:$dst, (node I64:$src))],
-                !strconcat("i64.", !strconcat(name, "\t$dst, $src")),
-                !strconcat("i64.", name), i64Inst>;
-}
-multiclass BinaryInt<SDNode node, string name, bits<32> i32Inst,
-                     bits<32> i64Inst> {
-  defm _I32 : I<(outs I32:$dst), (ins I32:$lhs, I32:$rhs), (outs), (ins),
-                [(set I32:$dst, (node I32:$lhs, I32:$rhs))],
-                !strconcat("i32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("i32.", name), i32Inst>;
-  defm _I64 : I<(outs I64:$dst), (ins I64:$lhs, I64:$rhs), (outs), (ins),
-                [(set I64:$dst, (node I64:$lhs, I64:$rhs))],
-                !strconcat("i64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("i64.", name), i64Inst>;
-}
-multiclass UnaryFP<SDNode node, string name, bits<32> f32Inst,
-                   bits<32> f64Inst> {
-  defm _F32 : I<(outs F32:$dst), (ins F32:$src), (outs), (ins),
-                [(set F32:$dst, (node F32:$src))],
-                !strconcat("f32.", !strconcat(name, "\t$dst, $src")),
-                !strconcat("f32.", name), f32Inst>;
-  defm _F64 : I<(outs F64:$dst), (ins F64:$src), (outs), (ins),
-                [(set F64:$dst, (node F64:$src))],
-                !strconcat("f64.", !strconcat(name, "\t$dst, $src")),
-                !strconcat("f64.", name), f64Inst>;
-}
-multiclass BinaryFP<SDNode node, string name, bits<32> f32Inst,
-                    bits<32> f64Inst> {
-  defm _F32 : I<(outs F32:$dst), (ins F32:$lhs, F32:$rhs), (outs), (ins),
-                [(set F32:$dst, (node F32:$lhs, F32:$rhs))],
-                !strconcat("f32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("f32.", name), f32Inst>;
-  defm _F64 : I<(outs F64:$dst), (ins F64:$lhs, F64:$rhs), (outs), (ins),
-                [(set F64:$dst, (node F64:$lhs, F64:$rhs))],
-                !strconcat("f64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("f64.", name), f64Inst>;
-}
-multiclass SIMDBinary<SDNode node, SDNode fnode, string name> {
-  defm _I8x16 : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
-                       (outs), (ins),
-                       [(set (v16i8 V128:$dst), (node V128:$lhs, V128:$rhs))],
-                       !strconcat("i8x16.",
-                         !strconcat(name, "\t$dst, $lhs, $rhs")),
-                       !strconcat("i8x16.", name)>;
-  defm _I16x8 : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
-                       (outs), (ins),
-                       [(set (v8i16 V128:$dst), (node V128:$lhs, V128:$rhs))],
-                       !strconcat("i16x8.",
-                         !strconcat(name, "\t$dst, $lhs, $rhs")),
-                       !strconcat("i16x8.", name)>;
-  defm _I32x4 : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
-                       (outs), (ins),
-                       [(set (v4i32 V128:$dst), (node V128:$lhs, V128:$rhs))],
-                       !strconcat("i32x4.",
-                         !strconcat(name, "\t$dst, $lhs, $rhs")),
-                       !strconcat("i32x4.", name)>;
-  defm _F32x4 : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
-                       (outs), (ins),
-                       [(set (v4f32 V128:$dst), (fnode V128:$lhs, V128:$rhs))],
-                       !strconcat("f32x4.",
-                         !strconcat(name, "\t$dst, $lhs, $rhs")),
-                       !strconcat("f32x4.", name)>;
-}
-multiclass ComparisonInt<CondCode cond, string name, bits<32> i32Inst, bits<32> i64Inst> {
-  defm _I32 : I<(outs I32:$dst), (ins I32:$lhs, I32:$rhs), (outs), (ins),
-                [(set I32:$dst, (setcc I32:$lhs, I32:$rhs, cond))],
-                !strconcat("i32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("i32.", name), i32Inst>;
-  defm _I64 : I<(outs I32:$dst), (ins I64:$lhs, I64:$rhs), (outs), (ins),
-                [(set I32:$dst, (setcc I64:$lhs, I64:$rhs, cond))],
-                !strconcat("i64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("i64.", name), i64Inst>;
-}
-multiclass ComparisonFP<CondCode cond, string name, bits<32> f32Inst, bits<32> f64Inst> {
-  defm _F32 : I<(outs I32:$dst), (ins F32:$lhs, F32:$rhs), (outs), (ins),
-                [(set I32:$dst, (setcc F32:$lhs, F32:$rhs, cond))],
-                !strconcat("f32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("f32.", name), f32Inst>;
-  defm  _F64 : I<(outs I32:$dst), (ins F64:$lhs, F64:$rhs), (outs), (ins),
-                [(set I32:$dst, (setcc F64:$lhs, F64:$rhs, cond))],
-                !strconcat("f64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
-                !strconcat("f64.", name), f64Inst>;
-}
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp b/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
index cd49bd1682ad..5efff32d6167 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
@@ -70,6 +70,8 @@ void WebAssemblyInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     CopyOpcode = WebAssembly::COPY_F32;
   else if (RC == &WebAssembly::F64RegClass)
     CopyOpcode = WebAssembly::COPY_F64;
+  else if (RC == &WebAssembly::V128RegClass)
+    CopyOpcode = WebAssembly::COPY_V128;
   else
     llvm_unreachable("Unexpected register class");
 
@@ -77,10 +79,8 @@ void WebAssemblyInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       .addReg(SrcReg, KillSrc ? RegState::Kill : 0);
 }
 
-MachineInstr *
-WebAssemblyInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
-                                             unsigned OpIdx1,
-                                             unsigned OpIdx2) const {
+MachineInstr *WebAssemblyInstrInfo::commuteInstructionImpl(
+    MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const {
   // If the operands are stackified, we can't reorder them.
   WebAssemblyFunctionInfo &MFI =
       *MI.getParent()->getParent()->getInfo<WebAssemblyFunctionInfo>();
@@ -165,12 +165,9 @@ unsigned WebAssemblyInstrInfo::removeBranch(MachineBasicBlock &MBB,
   return Count;
 }
 
-unsigned WebAssemblyInstrInfo::insertBranch(MachineBasicBlock &MBB,
-                                            MachineBasicBlock *TBB,
-                                            MachineBasicBlock *FBB,
-                                            ArrayRef<MachineOperand> Cond,
-                                            const DebugLoc &DL,
-                                            int *BytesAdded) const {
+unsigned WebAssemblyInstrInfo::insertBranch(
+    MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+    ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
   assert(!BytesAdded && "code size not handled");
 
   if (Cond.empty()) {
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
index aeb282a7febb..e3d795f2aab1 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -20,6 +20,9 @@ def HasAddr32 : Predicate<"!Subtarget->hasAddr64()">;
 def HasAddr64 : Predicate<"Subtarget->hasAddr64()">;
 def HasSIMD128 : Predicate<"Subtarget->hasSIMD128()">,
                            AssemblerPredicate<"FeatureSIMD128", "simd128">;
+def HasUnimplementedSIMD128 :
+    Predicate<"Subtarget->hasUnimplementedSIMD128()">,
+    AssemblerPredicate<"FeatureUnimplementedSIMD128", "unimplemented-simd128">;
 def HasAtomics : Predicate<"Subtarget->hasAtomics()">,
                            AssemblerPredicate<"FeatureAtomics", "atomics">;
 def HasNontrappingFPToInt :
@@ -64,6 +67,7 @@ def SDT_WebAssemblyArgument : SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>;
 def SDT_WebAssemblyReturn   : SDTypeProfile<0, -1, []>;
 def SDT_WebAssemblyWrapper  : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
                                                    SDTCisPtrTy<0>]>;
+def SDT_WebAssemblyThrow    : SDTypeProfile<0, 2, [SDTCisPtrTy<0>]>;
 
 //===----------------------------------------------------------------------===//
 // WebAssembly-specific DAG Nodes.
@@ -90,6 +94,8 @@ def WebAssemblyreturn   : SDNode<"WebAssemblyISD::RETURN",
                                  SDT_WebAssemblyReturn, [SDNPHasChain]>;
 def WebAssemblywrapper  : SDNode<"WebAssemblyISD::Wrapper",
                                  SDT_WebAssemblyWrapper>;
+def WebAssemblythrow : SDNode<"WebAssemblyISD::THROW", SDT_WebAssemblyThrow,
+                              [SDNPHasChain]>;
 
 //===----------------------------------------------------------------------===//
 // WebAssembly-specific Operands.
@@ -118,6 +124,18 @@ def f32imm_op : Operand<f32>;
 let OperandType = "OPERAND_F64IMM" in
 def f64imm_op : Operand<f64>;
 
+let OperandType = "OPERAND_VEC_I8IMM" in
+def vec_i8imm_op : Operand<i32>;
+
+let OperandType = "OPERAND_VEC_I16IMM" in
+def vec_i16imm_op : Operand<i32>;
+
+let OperandType = "OPERAND_VEC_I32IMM" in
+def vec_i32imm_op : Operand<i32>;
+
+let OperandType = "OPERAND_VEC_I64IMM" in
+def vec_i64imm_op : Operand<i64>;
+
 let OperandType = "OPERAND_FUNCTION32" in
 def function32_op : Operand<i32>;
 
@@ -128,6 +146,10 @@ let OperandType = "OPERAND_P2ALIGN" in {
 def P2Align : Operand<i32> {
   let PrintMethod = "printWebAssemblyP2AlignOperand";
 }
+
+let OperandType = "OPERAND_EVENT" in
+def event_op : Operand<i32>;
+
 } // OperandType = "OPERAND_P2ALIGN"
 
 let OperandType = "OPERAND_SIGNATURE" in {
@@ -142,6 +164,19 @@ def TypeIndex : Operand<i32>;
 } // OperandNamespace = "WebAssembly"
 
 //===----------------------------------------------------------------------===//
+// WebAssembly Register to Stack instruction mapping
+//===----------------------------------------------------------------------===//
+
+class StackRel;
+def getStackOpcode : InstrMapping {
+  let FilterClass = "StackRel";
+  let RowFields = ["BaseName"];
+  let ColFields = ["StackBased"];
+  let KeyCol = ["false"];
+  let ValueCols = [["true"]];
+}
+
+//===----------------------------------------------------------------------===//
 // WebAssembly Instruction Format Definitions.
 //===----------------------------------------------------------------------===//
 
@@ -151,74 +186,62 @@ include "WebAssemblyInstrFormats.td"
 // Additional instructions.
 //===----------------------------------------------------------------------===//
 
-multiclass ARGUMENT<WebAssemblyRegClass vt> {
-  let hasSideEffects = 1, Uses = [ARGUMENTS], isCodeGenOnly = 1 in
-  defm ARGUMENT_#vt : I<(outs vt:$res), (ins i32imm:$argno),
-                        (outs), (ins i32imm:$argno),
-                        [(set vt:$res, (WebAssemblyargument timm:$argno))]>;
+multiclass ARGUMENT<WebAssemblyRegClass reg, ValueType vt> {
+  let hasSideEffects = 1, isCodeGenOnly = 1,
+      Defs = []<Register>, Uses = [ARGUMENTS] in
+  defm ARGUMENT_#vt :
+    I<(outs reg:$res), (ins i32imm:$argno), (outs), (ins i32imm:$argno),
+      [(set (vt reg:$res), (WebAssemblyargument timm:$argno))]>;
 }
-multiclass SIMD_ARGUMENT<ValueType vt> {
-  let hasSideEffects = 1, Uses = [ARGUMENTS], isCodeGenOnly = 1 in
-  defm ARGUMENT_#vt : SIMD_I<(outs V128:$res), (ins i32imm:$argno),
-                             (outs), (ins i32imm:$argno),
-                             [(set (vt V128:$res),
-                                  (WebAssemblyargument timm:$argno))]>;
-}
-defm "": ARGUMENT<I32>;
-defm "": ARGUMENT<I64>;
-defm "": ARGUMENT<F32>;
-defm "": ARGUMENT<F64>;
-defm "": ARGUMENT<EXCEPT_REF>;
-defm "": SIMD_ARGUMENT<v16i8>;
-defm "": SIMD_ARGUMENT<v8i16>;
-defm "": SIMD_ARGUMENT<v4i32>;
-defm "": SIMD_ARGUMENT<v4f32>;
-
-let Defs = [ARGUMENTS] in {
+defm "": ARGUMENT<I32, i32>;
+defm "": ARGUMENT<I64, i64>;
+defm "": ARGUMENT<F32, f32>;
+defm "": ARGUMENT<F64, f64>;
+defm "": ARGUMENT<EXCEPT_REF, ExceptRef>;
 
-// get_local and set_local are not generated by instruction selection; they
+// local.get and local.set are not generated by instruction selection; they
 // are implied by virtual register uses and defs.
 multiclass LOCAL<WebAssemblyRegClass vt> {
 let hasSideEffects = 0 in {
-  // COPY is not an actual instruction in wasm, but since we allow get_local and
-  // set_local to be implicit during most of codegen, we can have a COPY which
-  // is actually a no-op because all the work is done in the implied get_local
-  // and set_local. COPYs are eliminated (and replaced with
-  // get_local/set_local) in the ExplicitLocals pass.
+  // COPY is not an actual instruction in wasm, but since we allow local.get and
+  // local.set to be implicit during most of codegen, we can have a COPY which
+  // is actually a no-op because all the work is done in the implied local.get
+  // and local.set. COPYs are eliminated (and replaced with
+  // local.get/local.set) in the ExplicitLocals pass.
   let isAsCheapAsAMove = 1, isCodeGenOnly = 1 in
   defm COPY_#vt : I<(outs vt:$res), (ins vt:$src), (outs), (ins), [],
-                    "copy_local\t$res, $src", "copy_local">;
+                    "local.copy\t$res, $src", "local.copy">;
 
   // TEE is similar to COPY, but writes two copies of its result. Typically
   // this would be used to stackify one result and write the other result to a
   // local.
   let isAsCheapAsAMove = 1, isCodeGenOnly = 1 in
   defm TEE_#vt : I<(outs vt:$res, vt:$also), (ins vt:$src), (outs), (ins), [],
-                   "tee_local\t$res, $also, $src", "tee_local">;
+                   "local.tee\t$res, $also, $src", "local.tee">;
 
-  // This is the actual get_local instruction in wasm. These are made explicit
+  // This is the actual local.get instruction in wasm. These are made explicit
   // by the ExplicitLocals pass. It has mayLoad because it reads from a wasm
   // local, which is a side effect not otherwise modeled in LLVM.
   let mayLoad = 1, isAsCheapAsAMove = 1 in
-  defm GET_LOCAL_#vt : I<(outs vt:$res), (ins local_op:$local),
+  defm LOCAL_GET_#vt : I<(outs vt:$res), (ins local_op:$local),
                          (outs), (ins local_op:$local), [],
-                         "get_local\t$res, $local", "get_local\t$local", 0x20>;
+                         "local.get\t$res, $local", "local.get\t$local", 0x20>;
 
-  // This is the actual set_local instruction in wasm. These are made explicit
+  // This is the actual local.set instruction in wasm. These are made explicit
   // by the ExplicitLocals pass. It has mayStore because it writes to a wasm
   // local, which is a side effect not otherwise modeled in LLVM.
   let mayStore = 1, isAsCheapAsAMove = 1 in
-  defm SET_LOCAL_#vt : I<(outs), (ins local_op:$local, vt:$src),
+  defm LOCAL_SET_#vt : I<(outs), (ins local_op:$local, vt:$src),
                          (outs), (ins local_op:$local), [],
-                         "set_local\t$local, $src", "set_local\t$local", 0x21>;
+                         "local.set\t$local, $src", "local.set\t$local", 0x21>;
 
-  // This is the actual tee_local instruction in wasm. TEEs are turned into
-  // TEE_LOCALs by the ExplicitLocals pass. It has mayStore for the same reason
-  // as SET_LOCAL.
+  // This is the actual local.tee instruction in wasm. TEEs are turned into
+  // LOCAL_TEEs by the ExplicitLocals pass. It has mayStore for the same reason
+  // as LOCAL_SET.
   let mayStore = 1, isAsCheapAsAMove = 1 in
-  defm TEE_LOCAL_#vt : I<(outs vt:$res), (ins local_op:$local, vt:$src),
+  defm LOCAL_TEE_#vt : I<(outs vt:$res), (ins local_op:$local, vt:$src),
                          (outs), (ins local_op:$local), [],
-                         "tee_local\t$res, $local, $src", "tee_local\t$local",
+                         "local.tee\t$res, $local, $src", "local.tee\t$local",
                          0x22>;
 
   // Unused values must be dropped in some contexts.
@@ -226,15 +249,15 @@ let hasSideEffects = 0 in {
                     "drop\t$src", "drop", 0x1a>;
 
   let mayLoad = 1 in
-  defm GET_GLOBAL_#vt : I<(outs vt:$res), (ins global_op:$local),
+  defm GLOBAL_GET_#vt : I<(outs vt:$res), (ins global_op:$local),
                           (outs), (ins global_op:$local), [],
-                          "get_global\t$res, $local", "get_global\t$local",
+                          "global.get\t$res, $local", "global.get\t$local",
                           0x23>;
 
   let mayStore = 1 in
-  defm SET_GLOBAL_#vt : I<(outs), (ins global_op:$local, vt:$src),
+  defm GLOBAL_SET_#vt : I<(outs), (ins global_op:$local, vt:$src),
                           (outs), (ins global_op:$local), [],
-                          "set_global\t$local, $src", "set_global\t$local",
+                          "global.set\t$local, $src", "global.set\t$local",
                           0x24>;
 
 } // hasSideEffects = 0
@@ -265,12 +288,12 @@ defm CONST_F64 : I<(outs F64:$res), (ins f64imm_op:$imm),
                    "f64.const\t$res, $imm", "f64.const\t$imm", 0x44>;
 } // isMoveImm = 1, isAsCheapAsAMove = 1, isReMaterializable = 1
 
-} // Defs = [ARGUMENTS]
-
 def : Pat<(i32 (WebAssemblywrapper tglobaladdr:$addr)),
           (CONST_I32 tglobaladdr:$addr)>;
 def : Pat<(i32 (WebAssemblywrapper texternalsym:$addr)),
           (CONST_I32 texternalsym:$addr)>;
+def : Pat<(i32 (WebAssemblywrapper mcsym:$sym)), (CONST_I32 mcsym:$sym)>;
+def : Pat<(i64 (WebAssemblywrapper mcsym:$sym)), (CONST_I64 mcsym:$sym)>;
 
 //===----------------------------------------------------------------------===//
 // Additional sets of instructions.
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInteger.td b/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
index f9f21fd1d754..d5b63d643697 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
@@ -12,7 +12,38 @@
 ///
 //===----------------------------------------------------------------------===//
 
-let Defs = [ARGUMENTS] in {
+multiclass UnaryInt<SDNode node, string name, bits<32> i32Inst,
+                    bits<32> i64Inst> {
+  defm _I32 : I<(outs I32:$dst), (ins I32:$src), (outs), (ins),
+                [(set I32:$dst, (node I32:$src))],
+                !strconcat("i32.", !strconcat(name, "\t$dst, $src")),
+                !strconcat("i32.", name), i32Inst>;
+  defm _I64 : I<(outs I64:$dst), (ins I64:$src), (outs), (ins),
+                [(set I64:$dst, (node I64:$src))],
+                !strconcat("i64.", !strconcat(name, "\t$dst, $src")),
+                !strconcat("i64.", name), i64Inst>;
+}
+multiclass BinaryInt<SDNode node, string name, bits<32> i32Inst,
+                     bits<32> i64Inst> {
+  defm _I32 : I<(outs I32:$dst), (ins I32:$lhs, I32:$rhs), (outs), (ins),
+                [(set I32:$dst, (node I32:$lhs, I32:$rhs))],
+                !strconcat("i32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("i32.", name), i32Inst>;
+  defm _I64 : I<(outs I64:$dst), (ins I64:$lhs, I64:$rhs), (outs), (ins),
+                [(set I64:$dst, (node I64:$lhs, I64:$rhs))],
+                !strconcat("i64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("i64.", name), i64Inst>;
+}
+multiclass ComparisonInt<CondCode cond, string name, bits<32> i32Inst, bits<32> i64Inst> {
+  defm _I32 : I<(outs I32:$dst), (ins I32:$lhs, I32:$rhs), (outs), (ins),
+                [(set I32:$dst, (setcc I32:$lhs, I32:$rhs, cond))],
+                !strconcat("i32.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("i32.", name), i32Inst>;
+  defm _I64 : I<(outs I32:$dst), (ins I64:$lhs, I64:$rhs), (outs), (ins),
+                [(set I32:$dst, (setcc I64:$lhs, I64:$rhs, cond))],
+                !strconcat("i64.", !strconcat(name, "\t$dst, $lhs, $rhs")),
+                !strconcat("i64.", name), i64Inst>;
+}
 
 // The spaces after the names are for aesthetic purposes only, to make
 // operands line up vertically after tab expansion.
@@ -63,16 +94,12 @@ defm EQZ_I64 : I<(outs I32:$dst), (ins I64:$src), (outs), (ins),
                  [(set I32:$dst, (setcc I64:$src, 0, SETEQ))],
                  "i64.eqz \t$dst, $src", "i64.eqz", 0x50>;
 
-} // Defs = [ARGUMENTS]
-
 // Optimize away an explicit mask on a rotate count.
 def : Pat<(rotl I32:$lhs, (and I32:$rhs, 31)), (ROTL_I32 I32:$lhs, I32:$rhs)>;
 def : Pat<(rotr I32:$lhs, (and I32:$rhs, 31)), (ROTR_I32 I32:$lhs, I32:$rhs)>;
 def : Pat<(rotl I64:$lhs, (and I64:$rhs, 63)), (ROTL_I64 I64:$lhs, I64:$rhs)>;
 def : Pat<(rotr I64:$lhs, (and I64:$rhs, 63)), (ROTR_I64 I64:$lhs, I64:$rhs)>;
 
-let Defs = [ARGUMENTS] in {
-
 defm SELECT_I32 : I<(outs I32:$dst), (ins I32:$lhs, I32:$rhs, I32:$cond),
                     (outs), (ins),
                     [(set I32:$dst, (select I32:$cond, I32:$lhs, I32:$rhs))],
@@ -82,8 +109,6 @@ defm SELECT_I64 : I<(outs I64:$dst), (ins I64:$lhs, I64:$rhs, I32:$cond),
                     [(set I64:$dst, (select I32:$cond, I64:$lhs, I64:$rhs))],
                     "i64.select\t$dst, $lhs, $rhs, $cond", "i64.select", 0x1b>;
 
-} // Defs = [ARGUMENTS]
-
 // ISD::SELECT requires its operand to conform to getBooleanContents, but
 // WebAssembly's select interprets any non-zero value as true, so we can fold
 // a setne with 0 into a select.
@@ -97,3 +122,10 @@ def : Pat<(select (i32 (seteq I32:$cond, 0)), I32:$lhs, I32:$rhs),
           (SELECT_I32 I32:$rhs, I32:$lhs, I32:$cond)>;
 def : Pat<(select (i32 (seteq I32:$cond, 0)), I64:$lhs, I64:$rhs),
           (SELECT_I64 I64:$rhs, I64:$lhs, I32:$cond)>;
+
+// The legalizer inserts an unnecessary `and 1` to make input conform
+// to getBooleanContents, which we can lower away.
+def : Pat<(select (i32 (and I32:$cond, 1)), I32:$lhs, I32:$rhs),
+          (SELECT_I32 I32:$lhs, I32:$rhs, I32:$cond)>;
+def : Pat<(select (i32 (and I32:$cond, 1)), I64:$lhs, I64:$rhs),
+          (SELECT_I64 I64:$lhs, I64:$rhs, I32:$cond)>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index 8a49325af2bd..518f81c61dc4 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -33,10 +33,8 @@ def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
     return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
 
-  KnownBits Known0;
-  CurDAG->computeKnownBits(N->getOperand(0), Known0, 0);
-  KnownBits Known1;
-  CurDAG->computeKnownBits(N->getOperand(1), Known1, 0);
+  KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
+  KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
   return (~Known0.Zero & ~Known1.Zero) == 0;
 }]>;
 
@@ -53,15 +51,14 @@ def regPlusGA : PatFrag<(ops node:$addr, node:$off),
 // We don't need a regPlusES because external symbols never have constant
 // offsets folded into them, so we can just use add.
 
-let Defs = [ARGUMENTS] in {
-
 // Defines atomic and non-atomic loads, regular and extending.
 multiclass WebAssemblyLoad<WebAssemblyRegClass rc, string Name, int Opcode> {
+  let mayLoad = 1 in
   defm "": I<(outs rc:$dst),
              (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
              (outs), (ins P2Align:$p2align, offset32_op:$off),
              [], !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}"),
-             !strconcat(Name, "\t${off}, ${p2align}"), Opcode>;
+             !strconcat(Name, "\t${off}${p2align}"), Opcode>;
 }
 
 // Basic load.
@@ -72,8 +69,6 @@ defm LOAD_I64 : WebAssemblyLoad<I64, "i64.load", 0x29>;
 defm LOAD_F32 : WebAssemblyLoad<F32, "f32.load", 0x2a>;
 defm LOAD_F64 : WebAssemblyLoad<F64, "f64.load", 0x2b>;
 
-} // Defs = [ARGUMENTS]
-
 // Select loads with no constant offset.
 class LoadPatNoOffset<ValueType ty, PatFrag kind, NI inst> :
   Pat<(ty (kind I32:$addr)), (inst 0, 0, I32:$addr)>;
@@ -143,8 +138,6 @@ def : LoadPatExternSymOffOnly<i64, load, LOAD_I64>;
 def : LoadPatExternSymOffOnly<f32, load, LOAD_F32>;
 def : LoadPatExternSymOffOnly<f64, load, LOAD_F64>;
 
-let Defs = [ARGUMENTS] in {
-
 // Extending load.
 defm LOAD8_S_I32 : WebAssemblyLoad<I32, "i32.load8_s", 0x2c>;
 defm LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.load8_u", 0x2d>;
@@ -157,8 +150,6 @@ defm LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.load16_u", 0x33>;
 defm LOAD32_S_I64 : WebAssemblyLoad<I64, "i64.load32_s", 0x34>;
 defm LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.load32_u", 0x35>;
 
-} // Defs = [ARGUMENTS]
-
 // Select extending loads with no constant offset.
 def : LoadPatNoOffset<i32, sextloadi8, LOAD8_S_I32>;
 def : LoadPatNoOffset<i32, zextloadi8, LOAD8_U_I32>;
@@ -302,17 +293,15 @@ def : LoadPatExternSymOffOnly<i64, extloadi8, LOAD8_U_I64>;
 def : LoadPatExternSymOffOnly<i64, extloadi16, LOAD16_U_I64>;
 def : LoadPatExternSymOffOnly<i64, extloadi32, LOAD32_U_I64>;
 
-
-let Defs = [ARGUMENTS] in {
-
 // Defines atomic and non-atomic stores, regular and truncating
 multiclass WebAssemblyStore<WebAssemblyRegClass rc, string Name, int Opcode> {
+  let mayStore = 1 in
   defm "" : I<(outs),
               (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val),
               (outs),
               (ins P2Align:$p2align, offset32_op:$off), [],
               !strconcat(Name, "\t${off}(${addr})${p2align}, $val"),
-              !strconcat(Name, "\t${off}, ${p2align}"), Opcode>;
+              !strconcat(Name, "\t${off}${p2align}"), Opcode>;
 }
 // Basic store.
 // Note: WebAssembly inverts SelectionDAG's usual operand order.
@@ -321,8 +310,6 @@ defm STORE_I64  : WebAssemblyStore<I64, "i64.store", 0x37>;
 defm STORE_F32  : WebAssemblyStore<F32, "f32.store", 0x38>;
 defm STORE_F64  : WebAssemblyStore<F64, "f64.store", 0x39>;
 
-} // Defs = [ARGUMENTS]
-
 // Select stores with no constant offset.
 class StorePatNoOffset<ValueType ty, PatFrag node, NI inst> :
   Pat<(node ty:$val, I32:$addr), (inst 0, 0, I32:$addr, ty:$val)>;
@@ -387,9 +374,6 @@ def : StorePatExternSymOffOnly<i64, store, STORE_I64>;
 def : StorePatExternSymOffOnly<f32, store, STORE_F32>;
 def : StorePatExternSymOffOnly<f64, store, STORE_F64>;
 
-
-let Defs = [ARGUMENTS] in {
-
 // Truncating store.
 defm STORE8_I32 : WebAssemblyStore<I32, "i32.store8", 0x3a>;
 defm STORE16_I32 : WebAssemblyStore<I32, "i32.store16", 0x3b>;
@@ -397,8 +381,6 @@ defm STORE8_I64 : WebAssemblyStore<I64, "i64.store8", 0x3c>;
 defm STORE16_I64 : WebAssemblyStore<I64, "i64.store16", 0x3d>;
 defm STORE32_I64 : WebAssemblyStore<I64, "i64.store32", 0x3e>;
 
-} // Defs = [ARGUMENTS]
-
 // Select truncating stores with no constant offset.
 def : StorePatNoOffset<i32, truncstorei8, STORE8_I32>;
 def : StorePatNoOffset<i32, truncstorei16, STORE16_I32>;
@@ -446,8 +428,6 @@ def : StorePatExternSymOffOnly<i64, truncstorei8, STORE8_I64>;
 def : StorePatExternSymOffOnly<i64, truncstorei16, STORE16_I64>;
 def : StorePatExternSymOffOnly<i64, truncstorei32, STORE32_I64>;
 
-let Defs = [ARGUMENTS] in {
-
 // Current memory size.
 defm MEMORY_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
                          (outs), (ins i32imm:$flags),
@@ -456,44 +436,13 @@ defm MEMORY_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
                          "memory.size\t$dst, $flags", "memory.size\t$flags",
                          0x3f>,
                        Requires<[HasAddr32]>;
-defm MEM_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
-                      (outs), (ins i32imm:$flags),
-                      [(set I32:$dst, (int_wasm_mem_size (i32 imm:$flags)))],
-                      "mem.size\t$dst, $flags", "mem.size\t$flags", 0x3f>,
-                    Requires<[HasAddr32]>;
-defm CURRENT_MEMORY_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
-                            (outs), (ins i32imm:$flags),
-                            [],
-                            "current_memory\t$dst",
-                            "current_memory\t$flags", 0x3f>,
-                          Requires<[HasAddr32]>;
 
 // Grow memory.
 defm MEMORY_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
-                         (outs), (ins i32imm:$flags, I32:$delta),
+                         (outs), (ins i32imm:$flags),
                          [(set I32:$dst,
                            (int_wasm_memory_grow (i32 imm:$flags),
                              I32:$delta))],
                          "memory.grow\t$dst, $flags, $delta",
-                         "memory.grow\t$flags, $delta", 0x3f>,
+                         "memory.grow\t$flags", 0x40>,
                        Requires<[HasAddr32]>;
-defm MEM_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
-                      (outs), (ins i32imm:$flags),
-                      [(set I32:$dst,
-                            (int_wasm_mem_grow (i32 imm:$flags), I32:$delta))],
-                      "mem.grow\t$dst, $flags, $delta", "mem.grow\t$flags",
-                      0x3f>,
-                    Requires<[HasAddr32]>;
-defm GROW_MEMORY_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
-                         (outs), (ins i32imm:$flags),
-                         [],
-                         "grow_memory\t$dst, $delta", "grow_memory\t$flags",
-                         0x40>,
-                       Requires<[HasAddr32]>;
-
-} // Defs = [ARGUMENTS]
-
-def : Pat<(int_wasm_current_memory),
-          (CURRENT_MEMORY_I32 0)>;
-def : Pat<(int_wasm_grow_memory I32:$delta),
-          (GROW_MEMORY_I32 0, $delta)>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 7d1edccdeb3c..587515c5b299 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -12,8 +12,796 @@
 ///
 //===----------------------------------------------------------------------===//
 
+// Instructions requiring HasSIMD128 and the simd128 prefix byte
+multiclass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
+                  list<dag> pattern_r, string asmstr_r = "",
+                  string asmstr_s = "", bits<32> simdop = -1> {
+  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
+              !or(0xfd00, !and(0xff, simdop))>,
+            Requires<[HasSIMD128]>;
+}
+
+defm "" : ARGUMENT<V128, v16i8>;
+defm "" : ARGUMENT<V128, v8i16>;
+defm "" : ARGUMENT<V128, v4i32>;
+defm "" : ARGUMENT<V128, v2i64>;
+defm "" : ARGUMENT<V128, v4f32>;
+defm "" : ARGUMENT<V128, v2f64>;
+
+// Constrained immediate argument types
+foreach SIZE = [8, 16] in
+def ImmI#SIZE : ImmLeaf<i32,
+  "return ((uint64_t)Imm & ((1UL << "#SIZE#") - 1)) == (uint64_t)Imm;"
+>;
+foreach SIZE = [2, 4, 8, 16, 32] in
+def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
+
+//===----------------------------------------------------------------------===//
+// Load and store
+//===----------------------------------------------------------------------===//
+
+// Load: v128.load
+multiclass SIMDLoad<ValueType vec_t> {
+  let mayLoad = 1 in
+  defm LOAD_#vec_t :
+    SIMD_I<(outs V128:$dst), (ins P2Align:$align, offset32_op:$off, I32:$addr),
+           (outs), (ins P2Align:$align, offset32_op:$off), [],
+           "v128.load\t$dst, ${off}(${addr})$align",
+           "v128.load\t$off$align", 0>;
+}
+
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
+defm "" : SIMDLoad<vec_t>;
+
+// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
+def : LoadPatNoOffset<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatImmOff<vec_t, load, regPlusImm, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatImmOff<vec_t, load, or_is_add, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatGlobalAddr<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatExternalSym<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatOffsetOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatGlobalAddrOffOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+def : LoadPatExternSymOffOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+}
+
+// Store: v128.store
+multiclass SIMDStore<ValueType vec_t> {
+  let mayStore = 1 in
+  defm STORE_#vec_t :
+    SIMD_I<(outs), (ins P2Align:$align, offset32_op:$off, I32:$addr, V128:$vec),
+           (outs), (ins P2Align:$align, offset32_op:$off), [],
+           "v128.store\t${off}(${addr})$align, $vec",
+           "v128.store\t$off$align", 1>;
+}
+
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
+defm "" : SIMDStore<vec_t>;
+
+// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
+def : StorePatNoOffset<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatImmOff<vec_t, store, regPlusImm, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatImmOff<vec_t, store, or_is_add, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatGlobalAddr<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatExternalSym<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatOffsetOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatGlobalAddrOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatExternSymOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+}
+
+//===----------------------------------------------------------------------===//
+// Constructing SIMD values
+//===----------------------------------------------------------------------===//
+
+// Constant: v128.const
+multiclass ConstVec<ValueType vec_t, dag ops, dag pat, string args> {
+  let isMoveImm = 1, isReMaterializable = 1,
+    Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+  defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops,
+                                  [(set V128:$dst, (vec_t pat))],
+                                  "v128.const\t$dst, "#args,
+                                  "v128.const\t"#args, 2>;
+}
+
+defm "" : ConstVec<v16i8,
+                   (ins vec_i8imm_op:$i0, vec_i8imm_op:$i1,
+                        vec_i8imm_op:$i2, vec_i8imm_op:$i3,
+                        vec_i8imm_op:$i4, vec_i8imm_op:$i5,
+                        vec_i8imm_op:$i6, vec_i8imm_op:$i7,
+                        vec_i8imm_op:$i8, vec_i8imm_op:$i9,
+                        vec_i8imm_op:$iA, vec_i8imm_op:$iB,
+                        vec_i8imm_op:$iC, vec_i8imm_op:$iD,
+                        vec_i8imm_op:$iE, vec_i8imm_op:$iF),
+                   (build_vector ImmI8:$i0, ImmI8:$i1, ImmI8:$i2, ImmI8:$i3,
+                                 ImmI8:$i4, ImmI8:$i5, ImmI8:$i6, ImmI8:$i7,
+                                 ImmI8:$i8, ImmI8:$i9, ImmI8:$iA, ImmI8:$iB,
+                                 ImmI8:$iC, ImmI8:$iD, ImmI8:$iE, ImmI8:$iF),
+                   !strconcat("$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7, ",
+                              "$i8, $i9, $iA, $iB, $iC, $iD, $iE, $iF")>;
+defm "" : ConstVec<v8i16,
+                   (ins vec_i16imm_op:$i0, vec_i16imm_op:$i1,
+                        vec_i16imm_op:$i2, vec_i16imm_op:$i3,
+                        vec_i16imm_op:$i4, vec_i16imm_op:$i5,
+                        vec_i16imm_op:$i6, vec_i16imm_op:$i7),
+                   (build_vector
+                     ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3,
+                     ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7),
+                   "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">;
+defm "" : ConstVec<v4i32,
+                   (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1,
+                        vec_i32imm_op:$i2, vec_i32imm_op:$i3),
+                   (build_vector (i32 imm:$i0), (i32 imm:$i1),
+                                 (i32 imm:$i2), (i32 imm:$i3)),
+                   "$i0, $i1, $i2, $i3">;
+defm "" : ConstVec<v2i64,
+                   (ins vec_i64imm_op:$i0, vec_i64imm_op:$i1),
+                   (build_vector (i64 imm:$i0), (i64 imm:$i1)),
+                   "$i0, $i1">;
+defm "" : ConstVec<v4f32,
+                   (ins f32imm_op:$i0, f32imm_op:$i1,
+                        f32imm_op:$i2, f32imm_op:$i3),
+                   (build_vector (f32 fpimm:$i0), (f32 fpimm:$i1),
+                                 (f32 fpimm:$i2), (f32 fpimm:$i3)),
+                   "$i0, $i1, $i2, $i3">;
+defm "" : ConstVec<v2f64,
+                  (ins f64imm_op:$i0, f64imm_op:$i1),
+                  (build_vector (f64 fpimm:$i0), (f64 fpimm:$i1)),
+                  "$i0, $i1">;
+
+// Shuffle lanes: shuffle
+defm SHUFFLE :
+  SIMD_I<(outs V128:$dst),
+         (ins V128:$x, V128:$y,
+           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
+           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
+           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
+           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
+           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
+           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
+           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
+           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
+         (outs),
+         (ins
+           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
+           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
+           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
+           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
+           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
+           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
+           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
+           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
+         [],
+         "v8x16.shuffle\t$dst, $x, $y, "#
+           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
+           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
+         "v8x16.shuffle\t"#
+           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
+           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
+         3>;
+
+// Shuffles after custom lowering
+def wasm_shuffle_t : SDTypeProfile<1, 18, []>;
+def wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>;
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
+def : Pat<(vec_t (wasm_shuffle (vec_t V128:$x), (vec_t V128:$y),
+            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
+            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
+            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
+            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
+            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
+            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
+            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
+            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF))),
+          (vec_t (SHUFFLE (vec_t V128:$x), (vec_t V128:$y),
+            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
+            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
+            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
+            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
+            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
+            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
+            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
+            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF)))>;
+}
+
+// Create vector with identical lanes: splat
+def splat2 : PatFrag<(ops node:$x), (build_vector node:$x, node:$x)>;
+def splat4 : PatFrag<(ops node:$x), (build_vector
+                       node:$x, node:$x, node:$x, node:$x)>;
+def splat8 : PatFrag<(ops node:$x), (build_vector
+                       node:$x, node:$x, node:$x, node:$x,
+                       node:$x, node:$x, node:$x, node:$x)>;
+def splat16 : PatFrag<(ops node:$x), (build_vector
+                        node:$x, node:$x, node:$x, node:$x,
+                        node:$x, node:$x, node:$x, node:$x,
+                        node:$x, node:$x, node:$x, node:$x,
+                        node:$x, node:$x, node:$x, node:$x)>;
+
+multiclass Splat<ValueType vec_t, string vec, WebAssemblyRegClass reg_t,
+                 PatFrag splat_pat, bits<32> simdop> {
+  // Prefer splats over v128.const for const splats (65 is lowest that works)
+  let AddedComplexity = 65 in
+  defm SPLAT_#vec_t : SIMD_I<(outs V128:$dst), (ins reg_t:$x), (outs), (ins),
+                             [(set (vec_t V128:$dst), (splat_pat reg_t:$x))],
+                             vec#".splat\t$dst, $x", vec#".splat", simdop>;
+}
+
+defm "" : Splat<v16i8, "i8x16", I32, splat16, 4>;
+defm "" : Splat<v8i16, "i16x8", I32, splat8, 8>;
+defm "" : Splat<v4i32, "i32x4", I32, splat4, 12>;
+defm "" : Splat<v2i64, "i64x2", I64, splat2, 15>;
+defm "" : Splat<v4f32, "f32x4", F32, splat4, 18>;
+defm "" : Splat<v2f64, "f64x2", F64, splat2, 21>;
+
+//===----------------------------------------------------------------------===//
+// Accessing lanes
+//===----------------------------------------------------------------------===//
+
+// Extract lane as a scalar: extract_lane / extract_lane_s / extract_lane_u
+multiclass ExtractLane<ValueType vec_t, string vec, ImmLeaf imm_t,
+                       WebAssemblyRegClass reg_t, bits<32> simdop,
+                       string suffix = "", SDNode extract = vector_extract> {
+  defm EXTRACT_LANE_#vec_t#suffix :
+      SIMD_I<(outs reg_t:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
+             (outs), (ins vec_i8imm_op:$idx),
+             [(set reg_t:$dst, (extract (vec_t V128:$vec), (i32 imm_t:$idx)))],
+             vec#".extract_lane"#suffix#"\t$dst, $vec, $idx",
+             vec#".extract_lane"#suffix#"\t$idx", simdop>;
+}
+
+multiclass ExtractPat<ValueType lane_t, int mask> {
+  def _s : PatFrag<(ops node:$vec, node:$idx),
+                   (i32 (sext_inreg
+                     (i32 (vector_extract
+                       node:$vec,
+                       node:$idx
+                     )),
+                     lane_t
+                   ))>;
+  def _u : PatFrag<(ops node:$vec, node:$idx),
+                   (i32 (and
+                     (i32 (vector_extract
+                       node:$vec,
+                       node:$idx
+                     )),
+                     (i32 mask)
+                   ))>;
+}
+
+defm extract_i8x16 : ExtractPat<i8, 0xff>;
+defm extract_i16x8 : ExtractPat<i16, 0xffff>;
+
+multiclass ExtractLaneExtended<string sign, bits<32> baseInst> {
+  defm "" : ExtractLane<v16i8, "i8x16", LaneIdx16, I32, baseInst, sign,
+                        !cast<PatFrag>("extract_i8x16"#sign)>;
+  defm "" : ExtractLane<v8i16, "i16x8", LaneIdx8, I32, !add(baseInst, 4), sign,
+                        !cast<PatFrag>("extract_i16x8"#sign)>;
+}
+
+defm "" : ExtractLaneExtended<"_s", 5>;
+let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+defm "" : ExtractLaneExtended<"_u", 6>;
+defm "" : ExtractLane<v4i32, "i32x4", LaneIdx4, I32, 13>;
+defm "" : ExtractLane<v2i64, "i64x2", LaneIdx2, I64, 16>;
+defm "" : ExtractLane<v4f32, "f32x4", LaneIdx4, F32, 19>;
+defm "" : ExtractLane<v2f64, "f64x2", LaneIdx2, F64, 22>;
+
+// It would be more conventional to use unsigned extracts, but v8
+// doesn't implement them yet
+def : Pat<(i32 (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx))),
+          (EXTRACT_LANE_v16i8_s V128:$vec, (i32 LaneIdx16:$idx))>;
+def : Pat<(i32 (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx))),
+          (EXTRACT_LANE_v8i16_s V128:$vec, (i32 LaneIdx8:$idx))>;
+
+// Lower undef lane indices to zero
+def : Pat<(and (i32 (vector_extract (v16i8 V128:$vec), undef)), (i32 0xff)),
+          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
+def : Pat<(and (i32 (vector_extract (v8i16 V128:$vec), undef)), (i32 0xffff)),
+          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
+def : Pat<(i32 (vector_extract (v16i8 V128:$vec), undef)),
+          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
+def : Pat<(i32 (vector_extract (v8i16 V128:$vec), undef)),
+          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
+def : Pat<(sext_inreg (i32 (vector_extract (v16i8 V128:$vec), undef)), i8),
+          (EXTRACT_LANE_v16i8_s V128:$vec, 0)>;
+def : Pat<(sext_inreg (i32 (vector_extract (v8i16 V128:$vec), undef)), i16),
+          (EXTRACT_LANE_v8i16_s V128:$vec, 0)>;
+def : Pat<(vector_extract (v4i32 V128:$vec), undef),
+          (EXTRACT_LANE_v4i32 V128:$vec, 0)>;
+def : Pat<(vector_extract (v2i64 V128:$vec), undef),
+          (EXTRACT_LANE_v2i64 V128:$vec, 0)>;
+def : Pat<(vector_extract (v4f32 V128:$vec), undef),
+          (EXTRACT_LANE_v4f32 V128:$vec, 0)>;
+def : Pat<(vector_extract (v2f64 V128:$vec), undef),
+          (EXTRACT_LANE_v2f64 V128:$vec, 0)>;
+
+// Replace lane value: replace_lane
+multiclass ReplaceLane<ValueType vec_t, string vec, ImmLeaf imm_t,
+                       WebAssemblyRegClass reg_t, ValueType lane_t,
+                       bits<32> simdop> {
+  defm REPLACE_LANE_#vec_t :
+      SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, reg_t:$x),
+             (outs), (ins vec_i8imm_op:$idx),
+             [(set V128:$dst, (vector_insert
+               (vec_t V128:$vec), (lane_t reg_t:$x), (i32 imm_t:$idx)))],
+             vec#".replace_lane\t$dst, $vec, $idx, $x",
+             vec#".replace_lane\t$idx", simdop>;
+}
+
+defm "" : ReplaceLane<v16i8, "i8x16", LaneIdx16, I32, i32, 7>;
+defm "" : ReplaceLane<v8i16, "i16x8", LaneIdx8, I32, i32, 11>;
+defm "" : ReplaceLane<v4i32, "i32x4", LaneIdx4, I32, i32, 14>;
+defm "" : ReplaceLane<v2i64, "i64x2", LaneIdx2, I64, i64, 17>;
+defm "" : ReplaceLane<v4f32, "f32x4", LaneIdx4, F32, f32, 20>;
+defm "" : ReplaceLane<v2f64, "f64x2", LaneIdx2, F64, f64, 23>;
+
+// Lower undef lane indices to zero
+def : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef),
+          (REPLACE_LANE_v16i8 V128:$vec, 0, I32:$x)>;
+def : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef),
+          (REPLACE_LANE_v8i16 V128:$vec, 0, I32:$x)>;
+def : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef),
+          (REPLACE_LANE_v4i32 V128:$vec, 0, I32:$x)>;
+def : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef),
+          (REPLACE_LANE_v2i64 V128:$vec, 0, I64:$x)>;
+def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
+          (REPLACE_LANE_v4f32 V128:$vec, 0, F32:$x)>;
+def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
+          (REPLACE_LANE_v2f64 V128:$vec, 0, F64:$x)>;
+
+// Arbitrary other BUILD_VECTOR patterns
+def : Pat<(v16i8 (build_vector
+            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
+            (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7),
+            (i32 I32:$x8), (i32 I32:$x9), (i32 I32:$x10), (i32 I32:$x11),
+            (i32 I32:$x12), (i32 I32:$x13), (i32 I32:$x14), (i32 I32:$x15)
+          )),
+          (v16i8 (REPLACE_LANE_v16i8
+            (v16i8 (REPLACE_LANE_v16i8
+              (v16i8 (REPLACE_LANE_v16i8
+                (v16i8 (REPLACE_LANE_v16i8
+                  (v16i8 (REPLACE_LANE_v16i8
+                    (v16i8 (REPLACE_LANE_v16i8
+                      (v16i8 (REPLACE_LANE_v16i8
+                        (v16i8 (REPLACE_LANE_v16i8
+                          (v16i8 (REPLACE_LANE_v16i8
+                            (v16i8 (REPLACE_LANE_v16i8
+                              (v16i8 (REPLACE_LANE_v16i8
+                                (v16i8 (REPLACE_LANE_v16i8
+                                  (v16i8 (REPLACE_LANE_v16i8
+                                    (v16i8 (REPLACE_LANE_v16i8
+                                      (v16i8 (REPLACE_LANE_v16i8
+                                        (v16i8 (SPLAT_v16i8 (i32 I32:$x0))),
+                                        1, I32:$x1
+                                      )),
+                                      2, I32:$x2
+                                    )),
+                                    3, I32:$x3
+                                  )),
+                                  4, I32:$x4
+                                )),
+                                5, I32:$x5
+                              )),
+                              6, I32:$x6
+                            )),
+                            7, I32:$x7
+                          )),
+                          8, I32:$x8
+                        )),
+                        9, I32:$x9
+                      )),
+                      10, I32:$x10
+                    )),
+                    11, I32:$x11
+                  )),
+                  12, I32:$x12
+                )),
+                13, I32:$x13
+              )),
+              14, I32:$x14
+            )),
+            15, I32:$x15
+          ))>;
+def : Pat<(v8i16 (build_vector
+            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
+            (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7)
+          )),
+          (v8i16 (REPLACE_LANE_v8i16
+            (v8i16 (REPLACE_LANE_v8i16
+              (v8i16 (REPLACE_LANE_v8i16
+                (v8i16 (REPLACE_LANE_v8i16
+                  (v8i16 (REPLACE_LANE_v8i16
+                    (v8i16 (REPLACE_LANE_v8i16
+                      (v8i16 (REPLACE_LANE_v8i16
+                        (v8i16 (SPLAT_v8i16 (i32 I32:$x0))),
+                        1, I32:$x1
+                      )),
+                      2, I32:$x2
+                    )),
+                    3, I32:$x3
+                  )),
+                  4, I32:$x4
+                )),
+                5, I32:$x5
+              )),
+              6, I32:$x6
+            )),
+            7, I32:$x7
+          ))>;
+def : Pat<(v4i32 (build_vector
+            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3)
+          )),
+          (v4i32 (REPLACE_LANE_v4i32
+            (v4i32 (REPLACE_LANE_v4i32
+              (v4i32 (REPLACE_LANE_v4i32
+                (v4i32 (SPLAT_v4i32 (i32 I32:$x0))),
+                1, I32:$x1
+              )),
+              2, I32:$x2
+            )),
+            3, I32:$x3
+          ))>;
+def : Pat<(v2i64 (build_vector (i64 I64:$x0), (i64 I64:$x1))),
+          (v2i64 (REPLACE_LANE_v2i64
+            (v2i64 (SPLAT_v2i64 (i64 I64:$x0))), 1, I64:$x1))>;
+def : Pat<(v4f32 (build_vector
+            (f32 F32:$x0), (f32 F32:$x1), (f32 F32:$x2), (f32 F32:$x3)
+          )),
+          (v4f32 (REPLACE_LANE_v4f32
+            (v4f32 (REPLACE_LANE_v4f32
+              (v4f32 (REPLACE_LANE_v4f32
+                (v4f32 (SPLAT_v4f32 (f32 F32:$x0))),
+                1, F32:$x1
+              )),
+              2, F32:$x2
+            )),
+            3, F32:$x3
+          ))>;
+def : Pat<(v2f64 (build_vector (f64 F64:$x0), (f64 F64:$x1))),
+          (v2f64 (REPLACE_LANE_v2f64
+            (v2f64 (SPLAT_v2f64 (f64 F64:$x0))), 1, F64:$x1))>;
+
+//===----------------------------------------------------------------------===//
+// Comparisons
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDCondition<ValueType vec_t, ValueType out_t, string vec,
+                         string name, CondCode cond, bits<32> simdop> {
+  defm _#vec_t :
+    SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
+           [(set (out_t V128:$dst),
+             (setcc (vec_t V128:$lhs), (vec_t V128:$rhs), cond)
+           )],
+           vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name, simdop>;
+}
+
+multiclass SIMDConditionInt<string name, CondCode cond, bits<32> baseInst> {
+  defm "" : SIMDCondition<v16i8, v16i8, "i8x16", name, cond, baseInst>;
+  defm "" : SIMDCondition<v8i16, v8i16, "i16x8", name, cond,
+                          !add(baseInst, 10)>;
+  defm "" : SIMDCondition<v4i32, v4i32, "i32x4", name, cond,
+                          !add(baseInst, 20)>;
+}
+
+multiclass SIMDConditionFP<string name, CondCode cond, bits<32> baseInst> {
+  defm "" : SIMDCondition<v4f32, v4i32, "f32x4", name, cond, baseInst>;
+  defm "" : SIMDCondition<v2f64, v2i64, "f64x2", name, cond,
+                          !add(baseInst, 6)>;
+}
+
+// Equality: eq
+let isCommutable = 1 in {
+defm EQ : SIMDConditionInt<"eq", SETEQ, 24>;
+defm EQ : SIMDConditionFP<"eq", SETOEQ, 64>;
+} // isCommutable = 1
+
+// Non-equality: ne
 let isCommutable = 1 in {
-defm ADD : SIMDBinary<add, fadd, "add ">;
-defm MUL: SIMDBinary<mul, fmul, "mul ">;
+defm NE : SIMDConditionInt<"ne", SETNE, 25>;
+defm NE : SIMDConditionFP<"ne", SETUNE, 65>;
 } // isCommutable = 1
-defm SUB: SIMDBinary<sub, fsub, "sub ">;
+
+// Less than: lt_s / lt_u / lt
+defm LT_S : SIMDConditionInt<"lt_s", SETLT, 26>;
+defm LT_U : SIMDConditionInt<"lt_u", SETULT, 27>;
+defm LT : SIMDConditionFP<"lt", SETOLT, 66>;
+
+// Greater than: gt_s / gt_u / gt
+defm GT_S : SIMDConditionInt<"gt_s", SETGT, 28>;
+defm GT_U : SIMDConditionInt<"gt_u", SETUGT, 29>;
+defm GT : SIMDConditionFP<"gt", SETOGT, 67>;
+
+// Less than or equal: le_s / le_u / le
+defm LE_S : SIMDConditionInt<"le_s", SETLE, 30>;
+defm LE_U : SIMDConditionInt<"le_u", SETULE, 31>;
+defm LE : SIMDConditionFP<"le", SETOLE, 68>;
+
+// Greater than or equal: ge_s / ge_u / ge
+defm GE_S : SIMDConditionInt<"ge_s", SETGE, 32>;
+defm GE_U : SIMDConditionInt<"ge_u", SETUGE, 33>;
+defm GE : SIMDConditionFP<"ge", SETOGE, 69>;
+
+// Lower float comparisons that don't care about NaN to standard WebAssembly
+// float comparisons. These instructions are generated in the target-independent
+// expansion of unordered comparisons and ordered ne.
+def : Pat<(v4i32 (seteq (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
+          (v4i32 (EQ_v4f32 (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;
+def : Pat<(v4i32 (setne (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
+          (v4i32 (NE_v4f32 (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;
+def : Pat<(v2i64 (seteq (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
+          (v2i64 (EQ_v2f64 (v2f64 V128:$lhs), (v2f64 V128:$rhs)))>;
+def : Pat<(v2i64 (setne (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
+          (v2i64 (NE_v2f64 (v2f64 V128:$lhs), (v2f64 V128:$rhs)))>;
+
+//===----------------------------------------------------------------------===//
+// Bitwise operations
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDBinary<ValueType vec_t, string vec, SDNode node, string name,
+                      bits<32> simdop> {
+  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
+                        (outs), (ins),
+                        [(set (vec_t V128:$dst),
+                          (node (vec_t V128:$lhs), (vec_t V128:$rhs))
+                        )],
+                        vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name,
+                        simdop>;
+}
+
+multiclass SIMDBitwise<SDNode node, string name, bits<32> simdop> {
+  defm "" : SIMDBinary<v16i8, "v128", node, name, simdop>;
+  defm "" : SIMDBinary<v8i16, "v128", node, name, simdop>;
+  defm "" : SIMDBinary<v4i32, "v128", node, name, simdop>;
+  defm "" : SIMDBinary<v2i64, "v128", node, name, simdop>;
+}
+
+multiclass SIMDUnary<ValueType vec_t, string vec, SDNode node, string name,
+                     bits<32> simdop> {
+  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
+                        [(set (vec_t V128:$dst),
+                          (vec_t (node (vec_t V128:$vec)))
+                        )],
+                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
+}
+
+// Bitwise logic: v128.not
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
+defm NOT: SIMDUnary<vec_t, "v128", vnot, "not", 76>;
+
+// Bitwise logic: v128.and / v128.or / v128.xor
+let isCommutable = 1 in {
+defm AND : SIMDBitwise<and, "and", 77>;
+defm OR : SIMDBitwise<or, "or", 78>;
+defm XOR : SIMDBitwise<xor, "xor", 79>;
+} // isCommutable = 1
+
+// Bitwise select: v128.bitselect
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
+  defm BITSELECT_#vec_t :
+    SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins),
+           [(set (vec_t V128:$dst),
+             (vec_t (int_wasm_bitselect
+               (vec_t V128:$v1), (vec_t V128:$v2), (vec_t V128:$c)
+             ))
+           )],
+           "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 80>;
+
+// Bitselect is equivalent to (c & v1) | (~c & v2)
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
+  def : Pat<(vec_t (or (and (vec_t V128:$c), (vec_t V128:$v1)),
+              (and (vnot V128:$c), (vec_t V128:$v2)))),
+            (!cast<Instruction>("BITSELECT_"#vec_t)
+              V128:$v1, V128:$v2, V128:$c)>;
+
+//===----------------------------------------------------------------------===//
+// Integer unary arithmetic
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDUnaryInt<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDUnary<v16i8, "i8x16", node, name, baseInst>;
+  defm "" : SIMDUnary<v8i16, "i16x8", node, name, !add(baseInst, 17)>;
+  defm "" : SIMDUnary<v4i32, "i32x4", node, name, !add(baseInst, 34)>;
+  defm "" : SIMDUnary<v2i64, "i64x2", node, name, !add(baseInst, 51)>;
+}
+
+multiclass SIMDReduceVec<ValueType vec_t, string vec, SDNode op, string name,
+                         bits<32> simdop> {
+  defm _#vec_t : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
+                        [(set I32:$dst, (i32 (op (vec_t V128:$vec))))],
+                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
+}
+
+multiclass SIMDReduce<SDNode op, string name, bits<32> baseInst> {
+  defm "" : SIMDReduceVec<v16i8, "i8x16", op, name, baseInst>;
+  defm "" : SIMDReduceVec<v8i16, "i16x8", op, name, !add(baseInst, 17)>;
+  defm "" : SIMDReduceVec<v4i32, "i32x4", op, name, !add(baseInst, 34)>;
+  defm "" : SIMDReduceVec<v2i64, "i64x2", op, name, !add(baseInst, 51)>;
+}
+
+// Integer vector negation
+def ivneg : PatFrag<(ops node:$in), (sub immAllZerosV, node:$in)>;
+
+// Integer negation: neg
+defm NEG : SIMDUnaryInt<ivneg, "neg", 81>;
+
+// Any lane true: any_true
+defm ANYTRUE : SIMDReduce<int_wasm_anytrue, "any_true", 82>;
+
+// All lanes true: all_true
+defm ALLTRUE : SIMDReduce<int_wasm_alltrue, "all_true", 83>;
+
+//===----------------------------------------------------------------------===//
+// Bit shifts
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDShift<ValueType vec_t, string vec, SDNode node, dag shift_vec,
+                     string name, bits<32> simdop> {
+  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x),
+                        (outs), (ins),
+                        [(set (vec_t V128:$dst),
+                          (node V128:$vec, (vec_t shift_vec)))],
+                        vec#"."#name#"\t$dst, $vec, $x", vec#"."#name, simdop>;
+}
+
+multiclass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDShift<v16i8, "i8x16", node, (splat16 I32:$x), name, baseInst>;
+  defm "" : SIMDShift<v8i16, "i16x8", node, (splat8 I32:$x), name,
+                      !add(baseInst, 17)>;
+  defm "" : SIMDShift<v4i32, "i32x4", node, (splat4 I32:$x), name,
+                      !add(baseInst, 34)>;
+  defm "" : SIMDShift<v2i64, "i64x2", node, (splat2 (i64 (zext I32:$x))),
+                      name, !add(baseInst, 51)>;
+}
+
+// Left shift by scalar: shl
+defm SHL : SIMDShiftInt<shl, "shl", 84>;
+
+// Right shift by scalar: shr_s / shr_u
+defm SHR_S : SIMDShiftInt<sra, "shr_s", 85>;
+defm SHR_U : SIMDShiftInt<srl, "shr_u", 86>;
+
+// Truncate i64 shift operands to i32s
+foreach shifts = [[shl, SHL_v2i64], [sra, SHR_S_v2i64], [srl, SHR_U_v2i64]] in
+def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), (v2i64 (splat2 I64:$x)))),
+          (v2i64 (shifts[1] (v2i64 V128:$vec), (I32_WRAP_I64 I64:$x)))>;
+
+// 2xi64 shifts with constant shift amounts are custom lowered to avoid wrapping
+def wasm_shift_t : SDTypeProfile<1, 2,
+  [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]
+>;
+def wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>;
+def wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>;
+def wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>;
+foreach shifts = [[wasm_shl, SHL_v2i64],
+                  [wasm_shr_s, SHR_S_v2i64],
+                  [wasm_shr_u, SHR_U_v2i64]] in
+def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), I32:$x)),
+          (v2i64 (shifts[1] (v2i64 V128:$vec), I32:$x))>;
+
+//===----------------------------------------------------------------------===//
+// Integer binary arithmetic
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDBinaryIntSmall<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDBinary<v16i8, "i8x16", node, name, baseInst>;
+  defm "" : SIMDBinary<v8i16, "i16x8", node, name, !add(baseInst, 17)>;
+}
+
+multiclass SIMDBinaryIntNoI64x2<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDBinaryIntSmall<node, name, baseInst>;
+  defm "" : SIMDBinary<v4i32, "i32x4", node, name, !add(baseInst, 34)>;
+}
+
+multiclass SIMDBinaryInt<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>;
+  defm "" : SIMDBinary<v2i64, "i64x2", node, name, !add(baseInst, 51)>;
+}
+
+// Integer addition: add / add_saturate_s / add_saturate_u
+let isCommutable = 1 in {
+defm ADD : SIMDBinaryInt<add, "add", 87>;
+defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_saturate_s", 88>;
+defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_saturate_u", 89>;
+} // isCommutable = 1
+
+// Integer subtraction: sub / sub_saturate_s / sub_saturate_u
+defm SUB : SIMDBinaryInt<sub, "sub", 90>;
+defm SUB_SAT_S :
+  SIMDBinaryIntSmall<int_wasm_sub_saturate_signed, "sub_saturate_s", 91>;
+defm SUB_SAT_U :
+  SIMDBinaryIntSmall<int_wasm_sub_saturate_unsigned, "sub_saturate_u", 92>;
+
+// Integer multiplication: mul
+defm MUL : SIMDBinaryIntNoI64x2<mul, "mul", 93>;
+
+//===----------------------------------------------------------------------===//
+// Floating-point unary arithmetic
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDUnaryFP<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDUnary<v4f32, "f32x4", node, name, baseInst>;
+  defm "" : SIMDUnary<v2f64, "f64x2", node, name, !add(baseInst, 11)>;
+}
+
+// Absolute value: abs
+defm ABS : SIMDUnaryFP<fabs, "abs", 149>;
+
+// Negation: neg
+defm NEG : SIMDUnaryFP<fneg, "neg", 150>;
+
+// Square root: sqrt
+let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 151>;
+
+//===----------------------------------------------------------------------===//
+// Floating-point binary arithmetic
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDBinaryFP<SDNode node, string name, bits<32> baseInst> {
+  defm "" : SIMDBinary<v4f32, "f32x4", node, name, baseInst>;
+  defm "" : SIMDBinary<v2f64, "f64x2", node, name, !add(baseInst, 11)>;
+}
+
+// Addition: add
+let isCommutable = 1 in
+defm ADD : SIMDBinaryFP<fadd, "add", 154>;
+
+// Subtraction: sub
+defm SUB : SIMDBinaryFP<fsub, "sub", 155>;
+
+// Multiplication: mul
+let isCommutable = 1 in
+defm MUL : SIMDBinaryFP<fmul, "mul", 156>;
+
+// Division: div
+let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+defm DIV : SIMDBinaryFP<fdiv, "div", 157>;
+
+// NaN-propagating minimum: min
+defm MIN : SIMDBinaryFP<fminimum, "min", 158>;
+
+// NaN-propagating maximum: max
+defm MAX : SIMDBinaryFP<fmaximum, "max", 159>;
+
+//===----------------------------------------------------------------------===//
+// Conversions
+//===----------------------------------------------------------------------===//
+
+multiclass SIMDConvert<ValueType vec_t, ValueType arg_t, SDNode op,
+                       string name, bits<32> simdop> {
+  defm op#_#vec_t#_#arg_t :
+    SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
+           [(set (vec_t V128:$dst), (vec_t (op (arg_t V128:$vec))))],
+           name#"\t$dst, $vec", name, simdop>;
+}
+
+// Integer to floating point: convert
+defm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_i32x4_s", 175>;
+defm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_i32x4_u", 176>;
+defm "" : SIMDConvert<v2f64, v2i64, sint_to_fp, "f64x2.convert_i64x2_s", 177>;
+defm "" : SIMDConvert<v2f64, v2i64, uint_to_fp, "f64x2.convert_i64x2_u", 178>;
+
+// Floating point to integer with saturation: trunc_sat
+defm "" : SIMDConvert<v4i32, v4f32, fp_to_sint, "i32x4.trunc_sat_f32x4_s", 171>;
+defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_f32x4_u", 172>;
+defm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_f64x2_s", 173>;
+defm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_f64x2_u", 174>;
+
+// Lower llvm.wasm.trunc.saturate.* to saturating instructions
+def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
+          (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>;
+def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
+          (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>;
+def : Pat<(v2i64 (int_wasm_trunc_saturate_signed (v2f64 V128:$src))),
+          (fp_to_sint_v2i64_v2f64 (v2f64 V128:$src))>;
+def : Pat<(v2i64 (int_wasm_trunc_saturate_unsigned (v2f64 V128:$src))),
+          (fp_to_uint_v2i64_v2f64 (v2f64 V128:$src))>;
+
+// Bitcasts are nops
+// Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
+foreach t1 = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
+foreach t2 = !foldl(
+  []<ValueType>, [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
+  acc, cur, !if(!eq(!cast<string>(t1), !cast<string>(cur)),
+    acc, !listconcat(acc, [cur])
+  )
+) in
+def : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>;
diff --git a/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp b/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
index e42dcbc0a8ac..ad838dfb574a 100644
--- a/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
@@ -31,6 +31,7 @@ class WebAssemblyLateEHPrepare final : public MachineFunctionPass {
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
+  bool removeUnnecessaryUnreachables(MachineFunction &MF);
   bool replaceFuncletReturns(MachineFunction &MF);
   bool hoistCatches(MachineFunction &MF);
   bool addCatchAlls(MachineFunction &MF);
@@ -47,7 +48,7 @@ public:
 
 char WebAssemblyLateEHPrepare::ID = 0;
 INITIALIZE_PASS(WebAssemblyLateEHPrepare, DEBUG_TYPE,
-                "WebAssembly Exception Preparation", false, false)
+                "WebAssembly Late Exception Preparation", false, false)
 
 FunctionPass *llvm::createWebAssemblyLateEHPrepare() {
   return new WebAssemblyLateEHPrepare();
@@ -59,7 +60,7 @@ FunctionPass *llvm::createWebAssemblyLateEHPrepare() {
 // possible search paths should be the same.
 // Returns nullptr in case it does not find any EH pad in the search, or finds
 // multiple different EH pads.
-MachineBasicBlock *GetMatchingEHPad(MachineInstr *MI) {
+static MachineBasicBlock *getMatchingEHPad(MachineInstr *MI) {
   MachineFunction *MF = MI->getParent()->getParent();
   SmallVector<MachineBasicBlock *, 2> WL;
   SmallPtrSet<MachineBasicBlock *, 2> Visited;
@@ -83,29 +84,35 @@ MachineBasicBlock *GetMatchingEHPad(MachineInstr *MI) {
   return EHPad;
 }
 
-// Erases the given BB and all its children from the function. If other BBs have
-// this BB as a successor, the successor relationships will be deleted as well.
-static void EraseBBAndChildren(MachineBasicBlock *MBB) {
-  SmallVector<MachineBasicBlock *, 8> WL;
-  WL.push_back(MBB);
+// Erase the specified BBs if the BB does not have any remaining predecessors,
+// and also all its dead children.
+template <typename Container>
+static void eraseDeadBBsAndChildren(const Container &MBBs) {
+  SmallVector<MachineBasicBlock *, 8> WL(MBBs.begin(), MBBs.end());
   while (!WL.empty()) {
     MachineBasicBlock *MBB = WL.pop_back_val();
-    for (auto *Pred : MBB->predecessors())
-      Pred->removeSuccessor(MBB);
-    for (auto *Succ : MBB->successors()) {
-      WL.push_back(Succ);
+    if (!MBB->pred_empty())
+      continue;
+    SmallVector<MachineBasicBlock *, 4> Succs(MBB->succ_begin(),
+                                              MBB->succ_end());
+    WL.append(MBB->succ_begin(), MBB->succ_end());
+    for (auto *Succ : Succs)
       MBB->removeSuccessor(Succ);
-    }
     MBB->eraseFromParent();
   }
 }
 
 bool WebAssemblyLateEHPrepare::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "********** Late EH Prepare **********\n"
+                       "********** Function: "
+                    << MF.getName() << '\n');
+
   if (MF.getTarget().getMCAsmInfo()->getExceptionHandlingType() !=
       ExceptionHandling::Wasm)
     return false;
 
   bool Changed = false;
+  Changed |= removeUnnecessaryUnreachables(MF);
   Changed |= addRethrows(MF);
   if (!MF.getFunction().hasPersonalityFn())
     return Changed;
@@ -118,6 +125,31 @@ bool WebAssemblyLateEHPrepare::runOnMachineFunction(MachineFunction &MF) {
   return Changed;
 }
 
+bool WebAssemblyLateEHPrepare::removeUnnecessaryUnreachables(
+    MachineFunction &MF) {
+  bool Changed = false;
+  for (auto &MBB : MF) {
+    for (auto &MI : MBB) {
+      if (!WebAssembly::isThrow(MI))
+        continue;
+      Changed = true;
+
+      // The instruction after the throw should be an unreachable or a branch to
+      // another BB that should eventually lead to an unreachable. Delete it
+      // because throw itself is a terminator, and also delete successors if
+      // any.
+      MBB.erase(std::next(MachineBasicBlock::iterator(MI)), MBB.end());
+      SmallVector<MachineBasicBlock *, 8> Succs(MBB.succ_begin(),
+                                                MBB.succ_end());
+      for (auto *Succ : Succs)
+        MBB.removeSuccessor(Succ);
+      eraseDeadBBsAndChildren(Succs);
+    }
+  }
+
+  return Changed;
+}
+
 bool WebAssemblyLateEHPrepare::replaceFuncletReturns(MachineFunction &MF) {
   bool Changed = false;
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
@@ -179,7 +211,7 @@ bool WebAssemblyLateEHPrepare::hoistCatches(MachineFunction &MF) {
         Catches.push_back(&MI);
 
   for (auto *Catch : Catches) {
-    MachineBasicBlock *EHPad = GetMatchingEHPad(Catch);
+    MachineBasicBlock *EHPad = getMatchingEHPad(Catch);
     assert(EHPad && "No matching EH pad for catch");
     if (EHPad->begin() == Catch)
       continue;
@@ -238,14 +270,18 @@ bool WebAssemblyLateEHPrepare::addRethrows(MachineFunction &MF) {
         Rethrow = BuildMI(MBB, InsertPt, MI.getDebugLoc(),
                           TII.get(WebAssembly::RETHROW_TO_CALLER));
 
-      // Becasue __cxa_rethrow does not return, the instruction after the
+      // Because __cxa_rethrow does not return, the instruction after the
       // rethrow should be an unreachable or a branch to another BB that should
       // eventually lead to an unreachable. Delete it because rethrow itself is
       // a terminator, and also delete non-EH pad successors if any.
       MBB.erase(std::next(MachineBasicBlock::iterator(Rethrow)), MBB.end());
+      SmallVector<MachineBasicBlock *, 8> NonPadSuccessors;
       for (auto *Succ : MBB.successors())
         if (!Succ->isEHPad())
-          EraseBBAndChildren(Succ);
+          NonPadSuccessors.push_back(Succ);
+      for (auto *Succ : NonPadSuccessors)
+        MBB.removeSuccessor(Succ);
+      eraseDeadBBsAndChildren(NonPadSuccessors);
     }
   return Changed;
 }
@@ -255,7 +291,7 @@ bool WebAssemblyLateEHPrepare::addRethrows(MachineFunction &MF) {
 //   %exn = catch 0
 //   call @__clang_call_terminate(%exn)
 //   unreachable
-// (There can be set_local and get_locals before the call if we didn't run
+// (There can be local.set and local.gets before the call if we didn't run
 // RegStackify)
 // But code transformations can change or add more control flow, so the call to
 // __clang_call_terminate() function may not be in the original EH pad anymore.
@@ -277,7 +313,7 @@ bool WebAssemblyLateEHPrepare::ensureSingleBBTermPads(MachineFunction &MF) {
 
   bool Changed = false;
   for (auto *Call : ClangCallTerminateCalls) {
-    MachineBasicBlock *EHPad = GetMatchingEHPad(Call);
+    MachineBasicBlock *EHPad = getMatchingEHPad(Call);
     assert(EHPad && "No matching EH pad for catch");
 
     // If it is already the form we want, skip it
@@ -294,7 +330,7 @@ bool WebAssemblyLateEHPrepare::ensureSingleBBTermPads(MachineFunction &MF) {
     // This runs after hoistCatches(), so catch instruction should be at the top
     assert(WebAssembly::isCatch(*Catch));
     // Takes the result register of the catch instruction as argument. There may
-    // have been some other set_local/get_locals in between, but at this point
+    // have been some other local.set/local.gets in between, but at this point
     // we don't care.
     Call->getOperand(1).setReg(Catch->getOperand(0).getReg());
     auto InsertPos = std::next(MachineBasicBlock::iterator(Catch));
@@ -302,8 +338,11 @@ bool WebAssemblyLateEHPrepare::ensureSingleBBTermPads(MachineFunction &MF) {
     BuildMI(*EHPad, InsertPos, Call->getDebugLoc(),
             TII.get(WebAssembly::UNREACHABLE));
     EHPad->erase(InsertPos, EHPad->end());
-    for (auto *Succ : EHPad->successors())
-      EraseBBAndChildren(Succ);
+    SmallVector<MachineBasicBlock *, 8> Succs(EHPad->succ_begin(),
+                                              EHPad->succ_end());
+    for (auto *Succ : Succs)
+      EHPad->removeSuccessor(Succ);
+    eraseDeadBBsAndChildren(Succs);
   }
   return Changed;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp b/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
index 5fb97e38939a..c9a3527d3fbd 100644
--- a/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
@@ -78,30 +78,102 @@ bool WebAssemblyLowerBrUnless::runOnMachineFunction(MachineFunction &MF) {
         MachineInstr *Def = MRI.getVRegDef(Cond);
         switch (Def->getOpcode()) {
           using namespace WebAssembly;
-        case EQ_I32: Def->setDesc(TII.get(NE_I32)); Inverted = true; break;
-        case NE_I32: Def->setDesc(TII.get(EQ_I32)); Inverted = true; break;
-        case GT_S_I32: Def->setDesc(TII.get(LE_S_I32)); Inverted = true; break;
-        case GE_S_I32: Def->setDesc(TII.get(LT_S_I32)); Inverted = true; break;
-        case LT_S_I32: Def->setDesc(TII.get(GE_S_I32)); Inverted = true; break;
-        case LE_S_I32: Def->setDesc(TII.get(GT_S_I32)); Inverted = true; break;
-        case GT_U_I32: Def->setDesc(TII.get(LE_U_I32)); Inverted = true; break;
-        case GE_U_I32: Def->setDesc(TII.get(LT_U_I32)); Inverted = true; break;
-        case LT_U_I32: Def->setDesc(TII.get(GE_U_I32)); Inverted = true; break;
-        case LE_U_I32: Def->setDesc(TII.get(GT_U_I32)); Inverted = true; break;
-        case EQ_I64: Def->setDesc(TII.get(NE_I64)); Inverted = true; break;
-        case NE_I64: Def->setDesc(TII.get(EQ_I64)); Inverted = true; break;
-        case GT_S_I64: Def->setDesc(TII.get(LE_S_I64)); Inverted = true; break;
-        case GE_S_I64: Def->setDesc(TII.get(LT_S_I64)); Inverted = true; break;
-        case LT_S_I64: Def->setDesc(TII.get(GE_S_I64)); Inverted = true; break;
-        case LE_S_I64: Def->setDesc(TII.get(GT_S_I64)); Inverted = true; break;
-        case GT_U_I64: Def->setDesc(TII.get(LE_U_I64)); Inverted = true; break;
-        case GE_U_I64: Def->setDesc(TII.get(LT_U_I64)); Inverted = true; break;
-        case LT_U_I64: Def->setDesc(TII.get(GE_U_I64)); Inverted = true; break;
-        case LE_U_I64: Def->setDesc(TII.get(GT_U_I64)); Inverted = true; break;
-        case EQ_F32: Def->setDesc(TII.get(NE_F32)); Inverted = true; break;
-        case NE_F32: Def->setDesc(TII.get(EQ_F32)); Inverted = true; break;
-        case EQ_F64: Def->setDesc(TII.get(NE_F64)); Inverted = true; break;
-        case NE_F64: Def->setDesc(TII.get(EQ_F64)); Inverted = true; break;
+        case EQ_I32:
+          Def->setDesc(TII.get(NE_I32));
+          Inverted = true;
+          break;
+        case NE_I32:
+          Def->setDesc(TII.get(EQ_I32));
+          Inverted = true;
+          break;
+        case GT_S_I32:
+          Def->setDesc(TII.get(LE_S_I32));
+          Inverted = true;
+          break;
+        case GE_S_I32:
+          Def->setDesc(TII.get(LT_S_I32));
+          Inverted = true;
+          break;
+        case LT_S_I32:
+          Def->setDesc(TII.get(GE_S_I32));
+          Inverted = true;
+          break;
+        case LE_S_I32:
+          Def->setDesc(TII.get(GT_S_I32));
+          Inverted = true;
+          break;
+        case GT_U_I32:
+          Def->setDesc(TII.get(LE_U_I32));
+          Inverted = true;
+          break;
+        case GE_U_I32:
+          Def->setDesc(TII.get(LT_U_I32));
+          Inverted = true;
+          break;
+        case LT_U_I32:
+          Def->setDesc(TII.get(GE_U_I32));
+          Inverted = true;
+          break;
+        case LE_U_I32:
+          Def->setDesc(TII.get(GT_U_I32));
+          Inverted = true;
+          break;
+        case EQ_I64:
+          Def->setDesc(TII.get(NE_I64));
+          Inverted = true;
+          break;
+        case NE_I64:
+          Def->setDesc(TII.get(EQ_I64));
+          Inverted = true;
+          break;
+        case GT_S_I64:
+          Def->setDesc(TII.get(LE_S_I64));
+          Inverted = true;
+          break;
+        case GE_S_I64:
+          Def->setDesc(TII.get(LT_S_I64));
+          Inverted = true;
+          break;
+        case LT_S_I64:
+          Def->setDesc(TII.get(GE_S_I64));
+          Inverted = true;
+          break;
+        case LE_S_I64:
+          Def->setDesc(TII.get(GT_S_I64));
+          Inverted = true;
+          break;
+        case GT_U_I64:
+          Def->setDesc(TII.get(LE_U_I64));
+          Inverted = true;
+          break;
+        case GE_U_I64:
+          Def->setDesc(TII.get(LT_U_I64));
+          Inverted = true;
+          break;
+        case LT_U_I64:
+          Def->setDesc(TII.get(GE_U_I64));
+          Inverted = true;
+          break;
+        case LE_U_I64:
+          Def->setDesc(TII.get(GT_U_I64));
+          Inverted = true;
+          break;
+        case EQ_F32:
+          Def->setDesc(TII.get(NE_F32));
+          Inverted = true;
+          break;
+        case NE_F32:
+          Def->setDesc(TII.get(EQ_F32));
+          Inverted = true;
+          break;
+        case EQ_F64:
+          Def->setDesc(TII.get(NE_F64));
+          Inverted = true;
+          break;
+        case NE_F64:
+          Def->setDesc(TII.get(EQ_F64));
+          Inverted = true;
+          break;
         case EQZ_I32: {
           // Invert an eqz by replacing it with its operand.
           Cond = Def->getOperand(1).getReg();
@@ -109,7 +181,8 @@ bool WebAssemblyLowerBrUnless::runOnMachineFunction(MachineFunction &MF) {
           Inverted = true;
           break;
         }
-        default: break;
+        default:
+          break;
         }
       }
 
diff --git a/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index e9cb7c10113b..0491f71cea7f 100644
--- a/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -50,24 +50,21 @@
 ///
 /// In detail, this pass does following things:
 ///
-/// 1) Create three global variables: __THREW__, __threwValue, and __tempRet0.
-///    __tempRet0 will be set within __cxa_find_matching_catch() function in
-///    JS library, and __THREW__ and __threwValue will be set in invoke wrappers
+/// 1) Assumes the existence of global variables: __THREW__, __threwValue
+///    __THREW__ and __threwValue will be set in invoke wrappers
 ///    in JS glue code. For what invoke wrappers are, refer to 3). These
 ///    variables are used for both exceptions and setjmp/longjmps.
 ///    __THREW__ indicates whether an exception or a longjmp occurred or not. 0
 ///    means nothing occurred, 1 means an exception occurred, and other numbers
 ///    mean a longjmp occurred. In the case of longjmp, __threwValue variable
 ///    indicates the corresponding setjmp buffer the longjmp corresponds to.
-///    In exception handling, __tempRet0 indicates the type of an exception
-///    caught, and in setjmp/longjmp, it means the second argument to longjmp
-///    function.
 ///
 /// * Exception handling
 ///
-/// 2) Create setThrew and setTempRet0 functions.
-///    The global variables created in 1) will exist in wasm address space,
-///    but their values should be set in JS code, so we provide these functions
+/// 2) We assume the existence of setThrew and setTempRet0/getTempRet0 functions
+///    at link time.
+///    The global variables in 1) will exist in wasm address space,
+///    but their values should be set in JS code, so these functions
 ///    as interfaces to JS glue code. These functions are equivalent to the
 ///    following JS functions, which actually exist in asm.js version of JS
 ///    library.
@@ -78,10 +75,12 @@
 ///        __threwValue = value;
 ///      }
 ///    }
+//
+///    setTempRet0 is called from __cxa_find_matching_catch() in JS glue code.
 ///
-///    function setTempRet0(value) {
-///      __tempRet0 = value;
-///    }
+///    In exception handling, getTempRet0 indicates the type of an exception
+///    caught, and in setjmp/longjmp, it means the second argument to longjmp
+///    function.
 ///
 /// 3) Lower
 ///      invoke @func(arg1, arg2) to label %invoke.cont unwind label %lpad
@@ -118,11 +117,10 @@
 ///      ... use %val ...
 ///    into
 ///      %fmc = call @__cxa_find_matching_catch_N(c1, c2, c3, ...)
-///      %val = {%fmc, __tempRet0}
+///      %val = {%fmc, getTempRet0()}
 ///      ... use %val ...
 ///    Here N is a number calculated based on the number of clauses.
-///    Global variable __tempRet0 is set within __cxa_find_matching_catch() in
-///    JS glue code.
+///    setTempRet0 is called from __cxa_find_matching_catch() in JS glue code.
 ///
 /// 5) Lower
 ///      resume {%a, %b}
@@ -138,7 +136,17 @@
 ///
 /// * Setjmp / Longjmp handling
 ///
-/// 7) In the function entry that calls setjmp, initialize setjmpTable and
+/// In case calls to longjmp() exists
+///
+/// 1) Lower
+///      longjmp(buf, value)
+///    into
+///      emscripten_longjmp_jmpbuf(buf, value)
+///    emscripten_longjmp_jmpbuf will be lowered to emscripten_longjmp later.
+///
+/// In case calls to setjmp() exists
+///
+/// 2) In the function entry that calls setjmp, initialize setjmpTable and
 ///    sejmpTableSize as follows:
 ///      setjmpTableSize = 4;
 ///      setjmpTable = (int *) malloc(40);
@@ -146,27 +154,22 @@
 ///    setjmpTable and setjmpTableSize are used in saveSetjmp() function in JS
 ///    code.
 ///
-/// 8) Lower
+/// 3) Lower
 ///      setjmp(buf)
 ///    into
 ///      setjmpTable = saveSetjmp(buf, label, setjmpTable, setjmpTableSize);
-///      setjmpTableSize = __tempRet0;
+///      setjmpTableSize = getTempRet0();
 ///    For each dynamic setjmp call, setjmpTable stores its ID (a number which
 ///    is incrementally assigned from 0) and its label (a unique number that
 ///    represents each callsite of setjmp). When we need more entries in
 ///    setjmpTable, it is reallocated in saveSetjmp() in JS code and it will
 ///    return the new table address, and assign the new table size in
-///    __tempRet0. saveSetjmp also stores the setjmp's ID into the buffer buf.
-///    A BB with setjmp is split into two after setjmp call in order to make the
-///    post-setjmp BB the possible destination of longjmp BB.
+///    setTempRet0(). saveSetjmp also stores the setjmp's ID into the buffer
+///    buf. A BB with setjmp is split into two after setjmp call in order to
+///    make the post-setjmp BB the possible destination of longjmp BB.
 ///
-/// 9) Lower
-///      longjmp(buf, value)
-///    into
-///      emscripten_longjmp_jmpbuf(buf, value)
-///    emscripten_longjmp_jmpbuf will be lowered to emscripten_longjmp later.
 ///
-/// 10) Lower every call that might longjmp into
+/// 4) Lower every call that might longjmp into
 ///      __THREW__ = 0;
 ///      call @__invoke_SIG(func, arg1, arg2)
 ///      %__THREW__.val = __THREW__;
@@ -176,32 +179,32 @@
 ///                            setjmpTableSize);
 ///        if (%label == 0)
 ///          emscripten_longjmp(%__THREW__.val, __threwValue);
-///        __tempRet0 = __threwValue;
+///        setTempRet0(__threwValue);
 ///      } else {
 ///        %label = -1;
 ///      }
-///      longjmp_result = __tempRet0;
+///      longjmp_result = getTempRet0();
 ///      switch label {
 ///        label 1: goto post-setjmp BB 1
 ///        label 2: goto post-setjmp BB 2
 ///        ...
 ///        default: goto splitted next BB
 ///      }
-///     testSetjmp examines setjmpTable to see if there is a matching setjmp
-///     call. After calling an invoke wrapper, if a longjmp occurred, __THREW__
-///     will be the address of matching jmp_buf buffer and __threwValue be the
-///     second argument to longjmp. mem[__THREW__.val] is a setjmp ID that is
-///     stored in saveSetjmp. testSetjmp returns a setjmp label, a unique ID to
-///     each setjmp callsite. Label 0 means this longjmp buffer does not
-///     correspond to one of the setjmp callsites in this function, so in this
-///     case we just chain the longjmp to the caller. (Here we call
-///     emscripten_longjmp, which is different from emscripten_longjmp_jmpbuf.
-///     emscripten_longjmp_jmpbuf takes jmp_buf as its first argument, while
-///     emscripten_longjmp takes an int. Both of them will eventually be lowered
-///     to emscripten_longjmp in s2wasm, but here we need two signatures - we
-///     can't translate an int value to a jmp_buf.)
-///     Label -1 means no longjmp occurred. Otherwise we jump to the right
-///     post-setjmp BB based on the label.
+///    testSetjmp examines setjmpTable to see if there is a matching setjmp
+///    call. After calling an invoke wrapper, if a longjmp occurred, __THREW__
+///    will be the address of matching jmp_buf buffer and __threwValue be the
+///    second argument to longjmp. mem[__THREW__.val] is a setjmp ID that is
+///    stored in saveSetjmp. testSetjmp returns a setjmp label, a unique ID to
+///    each setjmp callsite. Label 0 means this longjmp buffer does not
+///    correspond to one of the setjmp callsites in this function, so in this
+///    case we just chain the longjmp to the caller. (Here we call
+///    emscripten_longjmp, which is different from emscripten_longjmp_jmpbuf.
+///    emscripten_longjmp_jmpbuf takes jmp_buf as its first argument, while
+///    emscripten_longjmp takes an int. Both of them will eventually be lowered
+///    to emscripten_longjmp in s2wasm, but here we need two signatures - we
+///    can't translate an int value to a jmp_buf.)
+///    Label -1 means no longjmp occurred. Otherwise we jump to the right
+///    post-setjmp BB based on the label.
 ///
 ///===----------------------------------------------------------------------===//
 
@@ -239,7 +242,8 @@ class WebAssemblyLowerEmscriptenEHSjLj final : public ModulePass {
 
   GlobalVariable *ThrewGV;
   GlobalVariable *ThrewValueGV;
-  GlobalVariable *TempRet0GV;
+  Function *GetTempRet0Func;
+  Function *SetTempRet0Func;
   Function *ResumeF;
   Function *EHTypeIDF;
   Function *EmLongjmpF;
@@ -272,9 +276,6 @@ class WebAssemblyLowerEmscriptenEHSjLj final : public ModulePass {
   bool areAllExceptionsAllowed() const { return EHWhitelistSet.empty(); }
   bool canLongjmp(Module &M, const Value *Callee) const;
 
-  void createSetThrewFunction(Module &M);
-  void createSetTempRet0Function(Module &M);
-
   void rebuildSSA(Function &F);
 
 public:
@@ -282,9 +283,10 @@ public:
 
   WebAssemblyLowerEmscriptenEHSjLj(bool EnableEH = true, bool EnableSjLj = true)
       : ModulePass(ID), EnableEH(EnableEH), EnableSjLj(EnableSjLj),
-        ThrewGV(nullptr), ThrewValueGV(nullptr), TempRet0GV(nullptr),
-        ResumeF(nullptr), EHTypeIDF(nullptr), EmLongjmpF(nullptr),
-        EmLongjmpJmpbufF(nullptr), SaveSetjmpF(nullptr), TestSetjmpF(nullptr) {
+        ThrewGV(nullptr), ThrewValueGV(nullptr), GetTempRet0Func(nullptr),
+        SetTempRet0Func(nullptr), ResumeF(nullptr), EHTypeIDF(nullptr),
+        EmLongjmpF(nullptr), EmLongjmpJmpbufF(nullptr), SaveSetjmpF(nullptr),
+        TestSetjmpF(nullptr) {
     EHWhitelistSet.insert(EHWhitelist.begin(), EHWhitelist.end());
   }
   bool runOnModule(Module &M) override;
@@ -333,13 +335,15 @@ static bool canThrow(const Value *V) {
   return true;
 }
 
-static GlobalVariable *createGlobalVariableI32(Module &M, IRBuilder<> &IRB,
-                                               const char *Name) {
+// Get a global variable with the given name.  If it doesn't exist declare it,
+// which will generate an import and asssumes that it will exist at link time.
+static GlobalVariable *getGlobalVariableI32(Module &M, IRBuilder<> &IRB,
+                                            const char *Name) {
   if (M.getNamedGlobal(Name))
     report_fatal_error(Twine("variable name is reserved: ") + Name);
 
   return new GlobalVariable(M, IRB.getInt32Ty(), false,
-                            GlobalValue::WeakODRLinkage, IRB.getInt32(0), Name);
+                            GlobalValue::ExternalLinkage, nullptr, Name);
 }
 
 // Simple function name mangler.
@@ -508,7 +512,8 @@ bool WebAssemblyLowerEmscriptenEHSjLj::canLongjmp(Module &M,
   Function *ThrowF = M.getFunction("__cxa_throw");
   Function *TerminateF = M.getFunction("__clang_call_terminate");
   if (Callee == BeginCatchF || Callee == EndCatchF ||
-      Callee == AllocExceptionF || Callee == ThrowF || Callee == TerminateF)
+      Callee == AllocExceptionF || Callee == ThrowF || Callee == TerminateF ||
+      Callee == GetTempRet0Func || Callee == SetTempRet0Func)
     return false;
 
   // Otherwise we don't know
@@ -521,11 +526,11 @@ bool WebAssemblyLowerEmscriptenEHSjLj::canLongjmp(Module &M,
 //   %label = _testSetjmp(mem[%__THREW__.val], setjmpTable, setjmpTableSize);
 //   if (%label == 0)
 //     emscripten_longjmp(%__THREW__.val, threwValue);
-//   __tempRet0 = threwValue;
+//   setTempRet0(threwValue);
 // } else {
 //   %label = -1;
 // }
-// %longjmp_result = __tempRet0;
+// %longjmp_result = getTempRet0();
 //
 // As output parameters. returns %label, %longjmp_result, and the BB the last
 // instruction (%longjmp_result = ...) is in.
@@ -569,15 +574,15 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp(
   IRB.CreateCall(EmLongjmpF, {Threw, ThrewValue});
   IRB.CreateUnreachable();
 
-  // __tempRet0 = threwValue;
+  // setTempRet0(threwValue);
   IRB.SetInsertPoint(EndBB2);
-  IRB.CreateStore(ThrewValue, TempRet0GV);
+  IRB.CreateCall(SetTempRet0Func, ThrewValue);
   IRB.CreateBr(EndBB1);
 
   IRB.SetInsertPoint(ElseBB1);
   IRB.CreateBr(EndBB1);
 
-  // longjmp_result = __tempRet0;
+  // longjmp_result = getTempRet0();
   IRB.SetInsertPoint(EndBB1);
   PHINode *LabelPHI = IRB.CreatePHI(IRB.getInt32Ty(), 2, "label");
   LabelPHI->addIncoming(ThenLabel, EndBB2);
@@ -587,68 +592,7 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp(
   // Output parameter assignment
   Label = LabelPHI;
   EndBB = EndBB1;
-  LongjmpResult = IRB.CreateLoad(TempRet0GV, "longjmp_result");
-}
-
-// Create setThrew function
-// function setThrew(threw, value) {
-//   if (__THREW__ == 0) {
-//     __THREW__ = threw;
-//     __threwValue = value;
-//   }
-// }
-void WebAssemblyLowerEmscriptenEHSjLj::createSetThrewFunction(Module &M) {
-  LLVMContext &C = M.getContext();
-  IRBuilder<> IRB(C);
-
-  if (M.getNamedGlobal("setThrew"))
-    report_fatal_error("setThrew already exists");
-
-  Type *Params[] = {IRB.getInt32Ty(), IRB.getInt32Ty()};
-  FunctionType *FTy = FunctionType::get(IRB.getVoidTy(), Params, false);
-  Function *F =
-      Function::Create(FTy, GlobalValue::WeakODRLinkage, "setThrew", &M);
-  Argument *Arg1 = &*(F->arg_begin());
-  Argument *Arg2 = &*std::next(F->arg_begin());
-  Arg1->setName("threw");
-  Arg2->setName("value");
-  BasicBlock *EntryBB = BasicBlock::Create(C, "entry", F);
-  BasicBlock *ThenBB = BasicBlock::Create(C, "if.then", F);
-  BasicBlock *EndBB = BasicBlock::Create(C, "if.end", F);
-
-  IRB.SetInsertPoint(EntryBB);
-  Value *Threw = IRB.CreateLoad(ThrewGV, ThrewGV->getName() + ".val");
-  Value *Cmp = IRB.CreateICmpEQ(Threw, IRB.getInt32(0), "cmp");
-  IRB.CreateCondBr(Cmp, ThenBB, EndBB);
-
-  IRB.SetInsertPoint(ThenBB);
-  IRB.CreateStore(Arg1, ThrewGV);
-  IRB.CreateStore(Arg2, ThrewValueGV);
-  IRB.CreateBr(EndBB);
-
-  IRB.SetInsertPoint(EndBB);
-  IRB.CreateRetVoid();
-}
-
-// Create setTempRet0 function
-// function setTempRet0(value) {
-//   __tempRet0 = value;
-// }
-void WebAssemblyLowerEmscriptenEHSjLj::createSetTempRet0Function(Module &M) {
-  LLVMContext &C = M.getContext();
-  IRBuilder<> IRB(C);
-
-  if (M.getNamedGlobal("setTempRet0"))
-    report_fatal_error("setTempRet0 already exists");
-  Type *Params[] = {IRB.getInt32Ty()};
-  FunctionType *FTy = FunctionType::get(IRB.getVoidTy(), Params, false);
-  Function *F =
-      Function::Create(FTy, GlobalValue::WeakODRLinkage, "setTempRet0", &M);
-  F->arg_begin()->setName("value");
-  BasicBlock *EntryBB = BasicBlock::Create(C, "entry", F);
-  IRB.SetInsertPoint(EntryBB);
-  IRB.CreateStore(&*F->arg_begin(), TempRet0GV);
-  IRB.CreateRetVoid();
+  LongjmpResult = IRB.CreateCall(GetTempRet0Func, None, "longjmp_result");
 }
 
 void WebAssemblyLowerEmscriptenEHSjLj::rebuildSSA(Function &F) {
@@ -679,6 +623,8 @@ void WebAssemblyLowerEmscriptenEHSjLj::rebuildSSA(Function &F) {
 }
 
 bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "********** Lower Emscripten EH & SjLj **********\n");
+
   LLVMContext &C = M.getContext();
   IRBuilder<> IRB(C);
 
@@ -688,11 +634,19 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
   bool LongjmpUsed = LongjmpF && !LongjmpF->use_empty();
   bool DoSjLj = EnableSjLj && (SetjmpUsed || LongjmpUsed);
 
-  // Create global variables __THREW__, threwValue, and __tempRet0, which are
-  // used in common for both exception handling and setjmp/longjmp handling
-  ThrewGV = createGlobalVariableI32(M, IRB, "__THREW__");
-  ThrewValueGV = createGlobalVariableI32(M, IRB, "__threwValue");
-  TempRet0GV = createGlobalVariableI32(M, IRB, "__tempRet0");
+  // Declare (or get) global variables __THREW__, __threwValue, and
+  // getTempRet0/setTempRet0 function which are used in common for both
+  // exception handling and setjmp/longjmp handling
+  ThrewGV = getGlobalVariableI32(M, IRB, "__THREW__");
+  ThrewValueGV = getGlobalVariableI32(M, IRB, "__threwValue");
+  GetTempRet0Func =
+      Function::Create(FunctionType::get(IRB.getInt32Ty(), false),
+                       GlobalValue::ExternalLinkage, "getTempRet0", &M);
+  SetTempRet0Func = Function::Create(
+      FunctionType::get(IRB.getVoidTy(), IRB.getInt32Ty(), false),
+      GlobalValue::ExternalLinkage, "setTempRet0", &M);
+  GetTempRet0Func->setDoesNotThrow();
+  SetTempRet0Func->setDoesNotThrow();
 
   bool Changed = false;
 
@@ -721,22 +675,6 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
   if (DoSjLj) {
     Changed = true; // We have setjmp or longjmp somewhere
 
-    // Register saveSetjmp function
-    FunctionType *SetjmpFTy = SetjmpF->getFunctionType();
-    SmallVector<Type *, 4> Params = {SetjmpFTy->getParamType(0),
-                                     IRB.getInt32Ty(), Type::getInt32PtrTy(C),
-                                     IRB.getInt32Ty()};
-    FunctionType *FTy =
-        FunctionType::get(Type::getInt32PtrTy(C), Params, false);
-    SaveSetjmpF = Function::Create(FTy, GlobalValue::ExternalLinkage,
-                                   SaveSetjmpFName, &M);
-
-    // Register testSetjmp function
-    Params = {IRB.getInt32Ty(), Type::getInt32PtrTy(C), IRB.getInt32Ty()};
-    FTy = FunctionType::get(IRB.getInt32Ty(), Params, false);
-    TestSetjmpF = Function::Create(FTy, GlobalValue::ExternalLinkage,
-                                   TestSetjmpFName, &M);
-
     if (LongjmpF) {
       // Replace all uses of longjmp with emscripten_longjmp_jmpbuf, which is
       // defined in JS code
@@ -746,27 +684,43 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
 
       LongjmpF->replaceAllUsesWith(EmLongjmpJmpbufF);
     }
-    FTy = FunctionType::get(IRB.getVoidTy(),
-                            {IRB.getInt32Ty(), IRB.getInt32Ty()}, false);
-    EmLongjmpF =
-        Function::Create(FTy, GlobalValue::ExternalLinkage, EmLongjmpFName, &M);
 
-    // Only traverse functions that uses setjmp in order not to insert
-    // unnecessary prep / cleanup code in every function
-    SmallPtrSet<Function *, 8> SetjmpUsers;
-    for (User *U : SetjmpF->users()) {
-      auto *UI = cast<Instruction>(U);
-      SetjmpUsers.insert(UI->getFunction());
+    if (SetjmpF) {
+      // Register saveSetjmp function
+      FunctionType *SetjmpFTy = SetjmpF->getFunctionType();
+      SmallVector<Type *, 4> Params = {SetjmpFTy->getParamType(0),
+                                       IRB.getInt32Ty(), Type::getInt32PtrTy(C),
+                                       IRB.getInt32Ty()};
+      FunctionType *FTy =
+          FunctionType::get(Type::getInt32PtrTy(C), Params, false);
+      SaveSetjmpF = Function::Create(FTy, GlobalValue::ExternalLinkage,
+                                     SaveSetjmpFName, &M);
+
+      // Register testSetjmp function
+      Params = {IRB.getInt32Ty(), Type::getInt32PtrTy(C), IRB.getInt32Ty()};
+      FTy = FunctionType::get(IRB.getInt32Ty(), Params, false);
+      TestSetjmpF = Function::Create(FTy, GlobalValue::ExternalLinkage,
+                                     TestSetjmpFName, &M);
+
+      FTy = FunctionType::get(IRB.getVoidTy(),
+                              {IRB.getInt32Ty(), IRB.getInt32Ty()}, false);
+      EmLongjmpF = Function::Create(FTy, GlobalValue::ExternalLinkage,
+                                    EmLongjmpFName, &M);
+
+      // Only traverse functions that uses setjmp in order not to insert
+      // unnecessary prep / cleanup code in every function
+      SmallPtrSet<Function *, 8> SetjmpUsers;
+      for (User *U : SetjmpF->users()) {
+        auto *UI = cast<Instruction>(U);
+        SetjmpUsers.insert(UI->getFunction());
+      }
+      for (Function *F : SetjmpUsers)
+        runSjLjOnFunction(*F);
     }
-    for (Function *F : SetjmpUsers)
-      runSjLjOnFunction(*F);
   }
 
   if (!Changed) {
     // Delete unused global variables and functions
-    ThrewGV->eraseFromParent();
-    ThrewValueGV->eraseFromParent();
-    TempRet0GV->eraseFromParent();
     if (ResumeF)
       ResumeF->eraseFromParent();
     if (EHTypeIDF)
@@ -780,12 +734,6 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
     return false;
   }
 
-  // If we have made any changes while doing exception handling or
-  // setjmp/longjmp handling, we have to create these functions for JavaScript
-  // to call.
-  createSetThrewFunction(M);
-  createSetTempRet0Function(M);
-
   return true;
 }
 
@@ -908,8 +856,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runEHOnFunction(Function &F) {
     CallInst *FMCI = IRB.CreateCall(FMCF, FMCArgs, "fmc");
     Value *Undef = UndefValue::get(LPI->getType());
     Value *Pair0 = IRB.CreateInsertValue(Undef, FMCI, 0, "pair0");
-    Value *TempRet0 =
-        IRB.CreateLoad(TempRet0GV, TempRet0GV->getName() + ".val");
+    Value *TempRet0 = IRB.CreateCall(GetTempRet0Func, None, "tempret0");
     Value *Pair1 = IRB.CreateInsertValue(Pair0, TempRet0, 1, "pair1");
 
     LPI->replaceAllUsesWith(Pair1);
@@ -990,7 +937,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
     Instruction *NewSetjmpTable =
         IRB.CreateCall(SaveSetjmpF, Args, "setjmpTable");
     Instruction *NewSetjmpTableSize =
-        IRB.CreateLoad(TempRet0GV, "setjmpTableSize");
+        IRB.CreateCall(GetTempRet0Func, None, "setjmpTableSize");
     SetjmpTableInsts.push_back(NewSetjmpTable);
     SetjmpTableSizeInsts.push_back(NewSetjmpTableSize);
     ToErase.push_back(CI);
@@ -1098,7 +1045,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
 
   // Free setjmpTable buffer before each return instruction
   for (BasicBlock &BB : F) {
-    TerminatorInst *TI = BB.getTerminator();
+    Instruction *TI = BB.getTerminator();
     if (isa<ReturnInst>(TI))
       CallInst::CreateFree(SetjmpTable, TI);
   }
@@ -1112,7 +1059,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
   // ...
   // somebb:
   //   setjmpTable = saveSetjmp(buf, label, setjmpTable, setjmpTableSize);
-  //   setjmpTableSize = __tempRet0;
+  //   setjmpTableSize = getTempRet0();
   // So we need to make sure the SSA for these variables is valid so that every
   // saveSetjmp and testSetjmp calls have the correct arguments.
   SSAUpdater SetjmpTableSSA;
diff --git a/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp b/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
index ee708d637b25..84c877cb8d02 100644
--- a/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
@@ -18,15 +18,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssembly.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
 #include "llvm/Pass.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "wasm-lower-global-dtors"
@@ -59,6 +59,8 @@ ModulePass *llvm::createWebAssemblyLowerGlobalDtors() {
 }
 
 bool LowerGlobalDtors::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "********** Lower Global Destructors **********\n");
+
   GlobalVariable *GV = M.getGlobalVariable("llvm.global_dtors");
   if (!GV)
     return false;
@@ -77,18 +79,20 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
 
   // Collect the contents of @llvm.global_dtors, collated by priority and
   // associated symbol.
-  std::map<uint16_t, MapVector<Constant *, std::vector<Constant *> > > DtorFuncs;
+  std::map<uint16_t, MapVector<Constant *, std::vector<Constant *>>> DtorFuncs;
   for (Value *O : InitList->operands()) {
     ConstantStruct *CS = dyn_cast<ConstantStruct>(O);
-    if (!CS) continue; // Malformed.
+    if (!CS)
+      continue; // Malformed.
 
     ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
-    if (!Priority) continue; // Malformed.
+    if (!Priority)
+      continue; // Malformed.
     uint16_t PriorityValue = Priority->getLimitedValue(UINT16_MAX);
 
     Constant *DtorFunc = CS->getOperand(1);
     if (DtorFunc->isNullValue())
-      break;  // Found a null terminator, skip the rest.
+      break; // Found a null terminator, skip the rest.
 
     Constant *Associated = CS->getOperand(2);
     Associated = cast<Constant>(Associated->stripPointerCastsNoFollowAliases());
@@ -101,31 +105,23 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
   // extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
   LLVMContext &C = M.getContext();
   PointerType *VoidStar = Type::getInt8PtrTy(C);
-  Type *AtExitFuncArgs[] = { VoidStar };
-  FunctionType *AtExitFuncTy = FunctionType::get(
-          Type::getVoidTy(C),
-          AtExitFuncArgs,
-          /*isVarArg=*/false);
+  Type *AtExitFuncArgs[] = {VoidStar};
+  FunctionType *AtExitFuncTy =
+      FunctionType::get(Type::getVoidTy(C), AtExitFuncArgs,
+                        /*isVarArg=*/false);
 
-  Type *AtExitArgs[] = {
-    PointerType::get(AtExitFuncTy, 0),
-    VoidStar,
-    VoidStar
-  };
-  FunctionType *AtExitTy = FunctionType::get(
-          Type::getInt32Ty(C),
-          AtExitArgs,
-          /*isVarArg=*/false);
+  Type *AtExitArgs[] = {PointerType::get(AtExitFuncTy, 0), VoidStar, VoidStar};
+  FunctionType *AtExitTy = FunctionType::get(Type::getInt32Ty(C), AtExitArgs,
+                                             /*isVarArg=*/false);
   Constant *AtExit = M.getOrInsertFunction("__cxa_atexit", AtExitTy);
 
   // Declare __dso_local.
   Constant *DsoHandle = M.getNamedValue("__dso_handle");
   if (!DsoHandle) {
     Type *DsoHandleTy = Type::getInt8Ty(C);
-    GlobalVariable *Handle =
-        new GlobalVariable(M, DsoHandleTy, /*isConstant=*/true,
-                           GlobalVariable::ExternalWeakLinkage,
-                           nullptr, "__dso_handle");
+    GlobalVariable *Handle = new GlobalVariable(
+        M, DsoHandleTy, /*isConstant=*/true,
+        GlobalVariable::ExternalWeakLinkage, nullptr, "__dso_handle");
     Handle->setVisibility(GlobalVariable::HiddenVisibility);
     DsoHandle = Handle;
   }
@@ -139,13 +135,13 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
       Constant *Associated = AssociatedAndMore.first;
 
       Function *CallDtors = Function::Create(
-              AtExitFuncTy, Function::PrivateLinkage,
-              "call_dtors" +
-              (Priority != UINT16_MAX ?
-                 (Twine(".") + Twine(Priority)) : Twine()) +
-              (!Associated->isNullValue() ?
-                 (Twine(".") + Associated->getName()) : Twine()),
-              &M);
+          AtExitFuncTy, Function::PrivateLinkage,
+          "call_dtors" +
+              (Priority != UINT16_MAX ? (Twine(".") + Twine(Priority))
+                                      : Twine()) +
+              (!Associated->isNullValue() ? (Twine(".") + Associated->getName())
+                                          : Twine()),
+          &M);
       BasicBlock *BB = BasicBlock::Create(C, "body", CallDtors);
 
       for (auto Dtor : AssociatedAndMore.second)
@@ -155,29 +151,29 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
       FunctionType *VoidVoid = FunctionType::get(Type::getVoidTy(C),
                                                  /*isVarArg=*/false);
       Function *RegisterCallDtors = Function::Create(
-              VoidVoid, Function::PrivateLinkage,
-              "register_call_dtors" +
-              (Priority != UINT16_MAX ?
-                 (Twine(".") + Twine(Priority)) : Twine()) +
-              (!Associated->isNullValue() ?
-                 (Twine(".") + Associated->getName()) : Twine()),
-              &M);
+          VoidVoid, Function::PrivateLinkage,
+          "register_call_dtors" +
+              (Priority != UINT16_MAX ? (Twine(".") + Twine(Priority))
+                                      : Twine()) +
+              (!Associated->isNullValue() ? (Twine(".") + Associated->getName())
+                                          : Twine()),
+          &M);
       BasicBlock *EntryBB = BasicBlock::Create(C, "entry", RegisterCallDtors);
       BasicBlock *FailBB = BasicBlock::Create(C, "fail", RegisterCallDtors);
       BasicBlock *RetBB = BasicBlock::Create(C, "return", RegisterCallDtors);
 
       Value *Null = ConstantPointerNull::get(VoidStar);
-      Value *Args[] = { CallDtors, Null, DsoHandle };
+      Value *Args[] = {CallDtors, Null, DsoHandle};
       Value *Res = CallInst::Create(AtExit, Args, "call", EntryBB);
       Value *Cmp = new ICmpInst(*EntryBB, ICmpInst::ICMP_NE, Res,
                                 Constant::getNullValue(Res->getType()));
       BranchInst::Create(FailBB, RetBB, Cmp, EntryBB);
 
       // If `__cxa_atexit` hits out-of-memory, trap, so that we don't misbehave.
-      // This should be very rare, because if the process is running out of memory
-      // before main has even started, something is wrong.
-      CallInst::Create(Intrinsic::getDeclaration(&M, Intrinsic::trap),
-                       "", FailBB);
+      // This should be very rare, because if the process is running out of
+      // memory before main has even started, something is wrong.
+      CallInst::Create(Intrinsic::getDeclaration(&M, Intrinsic::trap), "",
+                       FailBB);
       new UnreachableInst(C, FailBB);
 
       ReturnInst::Create(C, RetBB);
diff --git a/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index d85db14fc679..fa862fbaa634 100644
--- a/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -30,6 +30,21 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
+// Defines llvm::WebAssembly::getStackOpcode to convert register instructions to
+// stack instructions
+#define GET_INSTRMAP_INFO 1
+#include "WebAssemblyGenInstrInfo.inc"
+
+// This disables the removal of registers when lowering into MC, as required
+// by some current tests.
+static cl::opt<bool>
+    WasmKeepRegisters("wasm-keep-registers", cl::Hidden,
+                      cl::desc("WebAssembly: output stack registers in"
+                               " instruction output for test purposes only."),
+                      cl::init(false));
+
+static void removeRegisterOperands(const MachineInstr *MI, MCInst &OutMI);
+
 MCSymbol *
 WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
   const GlobalValue *Global = MO.getGlobal();
@@ -40,35 +55,13 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
     const TargetMachine &TM = MF.getTarget();
     const Function &CurrentFunc = MF.getFunction();
 
-    SmallVector<wasm::ValType, 4> Returns;
-    SmallVector<wasm::ValType, 4> Params;
-
-    wasm::ValType iPTR =
-        MF.getSubtarget<WebAssemblySubtarget>().hasAddr64() ?
-        wasm::ValType::I64 :
-        wasm::ValType::I32;
-
-    SmallVector<MVT, 4> ResultMVTs;
-    ComputeLegalValueVTs(CurrentFunc, TM, FuncTy->getReturnType(), ResultMVTs);
-    // WebAssembly can't currently handle returning tuples.
-    if (ResultMVTs.size() <= 1)
-      for (MVT ResultMVT : ResultMVTs)
-        Returns.push_back(WebAssembly::toValType(ResultMVT));
-    else
-      Params.push_back(iPTR);
+    SmallVector<MVT, 1> ResultMVTs;
+    SmallVector<MVT, 4> ParamMVTs;
+    ComputeSignatureVTs(FuncTy, CurrentFunc, TM, ParamMVTs, ResultMVTs);
 
-    for (Type *Ty : FuncTy->params()) {
-      SmallVector<MVT, 4> ParamMVTs;
-      ComputeLegalValueVTs(CurrentFunc, TM, Ty, ParamMVTs);
-      for (MVT ParamMVT : ParamMVTs)
-        Params.push_back(WebAssembly::toValType(ParamMVT));
-    }
-
-    if (FuncTy->isVarArg())
-      Params.push_back(iPTR);
-
-    WasmSym->setReturns(std::move(Returns));
-    WasmSym->setParams(std::move(Params));
+    auto Signature = SignatureFromMVTs(ResultMVTs, ParamMVTs);
+    WasmSym->setSignature(Signature.get());
+    Printer.addSignature(std::move(Signature));
     WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
   }
 
@@ -82,10 +75,10 @@ MCSymbol *WebAssemblyMCInstLower::GetExternalSymbolSymbol(
       cast<MCSymbolWasm>(Printer.GetExternalSymbolSymbol(Name));
   const WebAssemblySubtarget &Subtarget = Printer.getSubtarget();
 
-  // __stack_pointer is a global variable; all other external symbols used by
-  // CodeGen are functions.  It's OK to hardcode knowledge of specific symbols
-  // here; this method is precisely there for fetching the signatures of known
-  // Clang-provided symbols.
+  // Except for the two exceptions (__stack_pointer and __cpp_exception), all
+  // other external symbols used by CodeGen are functions. It's OK to hardcode
+  // knowledge of specific symbols here; this method is precisely there for
+  // fetching the signatures of known Clang-provided symbols.
   if (strcmp(Name, "__stack_pointer") == 0) {
     WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
     WasmSym->setGlobalType(wasm::WasmGlobalType{
@@ -97,27 +90,55 @@ MCSymbol *WebAssemblyMCInstLower::GetExternalSymbolSymbol(
 
   SmallVector<wasm::ValType, 4> Returns;
   SmallVector<wasm::ValType, 4> Params;
-  GetSignature(Subtarget, Name, Returns, Params);
+  if (strcmp(Name, "__cpp_exception") == 0) {
+    WasmSym->setType(wasm::WASM_SYMBOL_TYPE_EVENT);
+    // We can't confirm its signature index for now because there can be
+    // imported exceptions. Set it to be 0 for now.
+    WasmSym->setEventType(
+        {wasm::WASM_EVENT_ATTRIBUTE_EXCEPTION, /* SigIndex */ 0});
+    // We may have multiple C++ compilation units to be linked together, each of
+    // which defines the exception symbol. To resolve them, we declare them as
+    // weak.
+    WasmSym->setWeak(true);
+    WasmSym->setExternal(true);
 
-  WasmSym->setReturns(std::move(Returns));
-  WasmSym->setParams(std::move(Params));
-  WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
+    // All C++ exceptions are assumed to have a single i32 (for wasm32) or i64
+    // (for wasm64) param type and void return type. The reaon is, all C++
+    // exception values are pointers, and to share the type section with
+    // functions, exceptions are assumed to have void return type.
+    Params.push_back(Subtarget.hasAddr64() ? wasm::ValType::I64
+                                           : wasm::ValType::I32);
+  } else { // Function symbols
+    WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
+    GetLibcallSignature(Subtarget, Name, Returns, Params);
+  }
+  auto Signature =
+      make_unique<wasm::WasmSignature>(std::move(Returns), std::move(Params));
+  WasmSym->setSignature(Signature.get());
+  Printer.addSignature(std::move(Signature));
 
   return WasmSym;
 }
 
 MCOperand WebAssemblyMCInstLower::LowerSymbolOperand(MCSymbol *Sym,
                                                      int64_t Offset,
-                                                     bool IsFunc) const {
+                                                     bool IsFunc, bool IsGlob,
+                                                     bool IsEvent) const {
   MCSymbolRefExpr::VariantKind VK =
       IsFunc ? MCSymbolRefExpr::VK_WebAssembly_FUNCTION
-             : MCSymbolRefExpr::VK_None;
+             : IsGlob ? MCSymbolRefExpr::VK_WebAssembly_GLOBAL
+                      : IsEvent ? MCSymbolRefExpr::VK_WebAssembly_EVENT
+                                : MCSymbolRefExpr::VK_None;
 
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, VK, Ctx);
 
   if (Offset != 0) {
     if (IsFunc)
       report_fatal_error("Function addresses with offsets not supported");
+    if (IsGlob)
+      report_fatal_error("Global indexes with offsets not supported");
+    if (IsEvent)
+      report_fatal_error("Event indexes with offsets not supported");
     Expr =
         MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(Offset, Ctx), Ctx);
   }
@@ -135,6 +156,8 @@ static wasm::ValType getType(const TargetRegisterClass *RC) {
     return wasm::ValType::F32;
   if (RC == &WebAssembly::F64RegClass)
     return wasm::ValType::F64;
+  if (RC == &WebAssembly::V128RegClass)
+    return wasm::ValType::V128;
   llvm_unreachable("Unexpected register class");
 }
 
@@ -187,8 +210,10 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
             Params.pop_back();
 
           MCSymbolWasm *WasmSym = cast<MCSymbolWasm>(Sym);
-          WasmSym->setReturns(std::move(Returns));
-          WasmSym->setParams(std::move(Params));
+          auto Signature = make_unique<wasm::WasmSignature>(std::move(Returns),
+                                                            std::move(Params));
+          WasmSym->setSignature(Signature.get());
+          Printer.addSignature(std::move(Signature));
           WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
 
           const MCExpr *Expr = MCSymbolRefExpr::create(
@@ -212,21 +237,68 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
       break;
     }
     case MachineOperand::MO_GlobalAddress:
-      assert(MO.getTargetFlags() == 0 &&
+      assert(MO.getTargetFlags() == WebAssemblyII::MO_NO_FLAG &&
              "WebAssembly does not use target flags on GlobalAddresses");
       MCOp = LowerSymbolOperand(GetGlobalAddressSymbol(MO), MO.getOffset(),
-                                MO.getGlobal()->getValueType()->isFunctionTy());
+                                MO.getGlobal()->getValueType()->isFunctionTy(),
+                                false, false);
       break;
     case MachineOperand::MO_ExternalSymbol:
       // The target flag indicates whether this is a symbol for a
       // variable or a function.
-      assert((MO.getTargetFlags() & -2) == 0 &&
-             "WebAssembly uses only one target flag bit on ExternalSymbols");
-      MCOp = LowerSymbolOperand(GetExternalSymbolSymbol(MO), /*Offset=*/0,
-                                MO.getTargetFlags() & 1);
+      assert((MO.getTargetFlags() & ~WebAssemblyII::MO_SYMBOL_MASK) == 0 &&
+             "WebAssembly uses only symbol flags on ExternalSymbols");
+      MCOp = LowerSymbolOperand(
+          GetExternalSymbolSymbol(MO), /*Offset=*/0,
+          (MO.getTargetFlags() & WebAssemblyII::MO_SYMBOL_FUNCTION) != 0,
+          (MO.getTargetFlags() & WebAssemblyII::MO_SYMBOL_GLOBAL) != 0,
+          (MO.getTargetFlags() & WebAssemblyII::MO_SYMBOL_EVENT) != 0);
+      break;
+    case MachineOperand::MO_MCSymbol:
+      // This is currently used only for LSDA symbols (GCC_except_table),
+      // because global addresses or other external symbols are handled above.
+      assert(MO.getTargetFlags() == 0 &&
+             "WebAssembly does not use target flags on MCSymbol");
+      MCOp = LowerSymbolOperand(MO.getMCSymbol(), /*Offset=*/0, false, false,
+                                false);
       break;
     }
 
     OutMI.addOperand(MCOp);
   }
+
+  if (!WasmKeepRegisters)
+    removeRegisterOperands(MI, OutMI);
+}
+
+static void removeRegisterOperands(const MachineInstr *MI, MCInst &OutMI) {
+  // Remove all uses of stackified registers to bring the instruction format
+  // into its final stack form used thruout MC, and transition opcodes to
+  // their _S variant.
+  // We do this seperate from the above code that still may need these
+  // registers for e.g. call_indirect signatures.
+  // See comments in lib/Target/WebAssembly/WebAssemblyInstrFormats.td for
+  // details.
+  // TODO: the code above creates new registers which are then removed here.
+  // That code could be slightly simplified by not doing that, though maybe
+  // it is simpler conceptually to keep the code above in "register mode"
+  // until this transition point.
+  // FIXME: we are not processing inline assembly, which contains register
+  // operands, because it is used by later target generic code.
+  if (MI->isDebugInstr() || MI->isLabel() || MI->isInlineAsm())
+    return;
+
+  // Transform to _S instruction.
+  auto RegOpcode = OutMI.getOpcode();
+  auto StackOpcode = WebAssembly::getStackOpcode(RegOpcode);
+  assert(StackOpcode != -1 && "Failed to stackify instruction");
+  OutMI.setOpcode(StackOpcode);
+
+  // Remove register operands.
+  for (auto I = OutMI.getNumOperands(); I; --I) {
+    auto &MO = OutMI.getOperand(I - 1);
+    if (MO.isReg()) {
+      OutMI.erase(&MO);
+    }
+  }
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyMCInstLower.h b/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
index 41b4313bb38c..fa7a0ea61b3b 100644
--- a/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
+++ b/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
@@ -33,8 +33,8 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyMCInstLower {
 
   MCSymbol *GetGlobalAddressSymbol(const MachineOperand &MO) const;
   MCSymbol *GetExternalSymbolSymbol(const MachineOperand &MO) const;
-  MCOperand LowerSymbolOperand(MCSymbol *Sym, int64_t Offset,
-                               bool IsFunc) const;
+  MCOperand LowerSymbolOperand(MCSymbol *Sym, int64_t Offset, bool IsFunc,
+                               bool IsGlob, bool IsEvent) const;
 
 public:
   WebAssemblyMCInstLower(MCContext &ctx, WebAssemblyAsmPrinter &printer)
diff --git a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
index e511e574050f..0157af0f8510 100644
--- a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
@@ -43,20 +43,38 @@ void llvm::ComputeLegalValueVTs(const Function &F, const TargetMachine &TM,
   }
 }
 
-void llvm::ComputeSignatureVTs(const Function &F, const TargetMachine &TM,
+void llvm::ComputeSignatureVTs(const FunctionType *Ty, const Function &F,
+                               const TargetMachine &TM,
                                SmallVectorImpl<MVT> &Params,
                                SmallVectorImpl<MVT> &Results) {
-  ComputeLegalValueVTs(F, TM, F.getReturnType(), Results);
+  ComputeLegalValueVTs(F, TM, Ty->getReturnType(), Results);
 
+  MVT PtrVT = MVT::getIntegerVT(TM.createDataLayout().getPointerSizeInBits());
   if (Results.size() > 1) {
     // WebAssembly currently can't lower returns of multiple values without
     // demoting to sret (see WebAssemblyTargetLowering::CanLowerReturn). So
     // replace multiple return values with a pointer parameter.
     Results.clear();
-    Params.push_back(
-        MVT::getIntegerVT(TM.createDataLayout().getPointerSizeInBits()));
+    Params.push_back(PtrVT);
   }
 
-  for (auto &Arg : F.args())
-    ComputeLegalValueVTs(F, TM, Arg.getType(), Params);
+  for (auto *Param : Ty->params())
+    ComputeLegalValueVTs(F, TM, Param, Params);
+  if (Ty->isVarArg())
+    Params.push_back(PtrVT);
+}
+
+void llvm::ValTypesFromMVTs(const ArrayRef<MVT> &In,
+                            SmallVectorImpl<wasm::ValType> &Out) {
+  for (MVT Ty : In)
+    Out.push_back(WebAssembly::toValType(Ty));
+}
+
+std::unique_ptr<wasm::WasmSignature>
+llvm::SignatureFromMVTs(const SmallVectorImpl<MVT> &Results,
+                        const SmallVectorImpl<MVT> &Params) {
+  auto Sig = make_unique<wasm::WasmSignature>();
+  ValTypesFromMVTs(Results, Sig->Returns);
+  ValTypesFromMVTs(Params, Sig->Params);
+  return Sig;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
index a60b10fc5309..4be4beb85d04 100644
--- a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
@@ -17,7 +17,9 @@
 #define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYMACHINEFUNCTIONINFO_H
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/MC/MCSymbolWasm.h"
 
 namespace llvm {
 
@@ -50,7 +52,7 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
   // overaligned values on the user stack.
   unsigned BasePtrVreg = -1U;
 
- public:
+public:
   explicit WebAssemblyFunctionInfo(MachineFunction &MF) : MF(MF) {}
   ~WebAssemblyFunctionInfo() override;
 
@@ -60,7 +62,10 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
   void addResult(MVT VT) { Results.push_back(VT); }
   const std::vector<MVT> &getResults() const { return Results; }
 
-  void clearParamsAndResults() { Params.clear(); Results.clear(); }
+  void clearParamsAndResults() {
+    Params.clear();
+    Results.clear();
+  }
 
   void setNumLocals(size_t NumLocals) { Locals.resize(NumLocals, MVT::i32); }
   void setLocal(size_t i, MVT VT) { Locals[i] = VT; }
@@ -115,13 +120,22 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
   }
 };
 
-void ComputeLegalValueVTs(const Function &F, const TargetMachine &TM,
-                          Type *Ty, SmallVectorImpl<MVT> &ValueVTs);
+void ComputeLegalValueVTs(const Function &F, const TargetMachine &TM, Type *Ty,
+                          SmallVectorImpl<MVT> &ValueVTs);
 
-void ComputeSignatureVTs(const Function &F, const TargetMachine &TM,
-                         SmallVectorImpl<MVT> &Params,
+// Compute the signature for a given FunctionType (Ty). Note that it's not the
+// signature for F (F is just used to get varous context)
+void ComputeSignatureVTs(const FunctionType *Ty, const Function &F,
+                         const TargetMachine &TM, SmallVectorImpl<MVT> &Params,
                          SmallVectorImpl<MVT> &Results);
 
+void ValTypesFromMVTs(const ArrayRef<MVT> &In,
+                      SmallVectorImpl<wasm::ValType> &Out);
+
+std::unique_ptr<wasm::WasmSignature>
+SignatureFromMVTs(const SmallVectorImpl<MVT> &Results,
+                  const SmallVectorImpl<MVT> &Params);
+
 } // end namespace llvm
 
 #endif
diff --git a/lib/Target/WebAssembly/WebAssemblyStoreResults.cpp b/lib/Target/WebAssembly/WebAssemblyMemIntrinsicResults.cpp
index 893e8484c4c6..c4b5e96db0c7 100644
--- a/lib/Target/WebAssembly/WebAssemblyStoreResults.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyMemIntrinsicResults.cpp
@@ -1,4 +1,4 @@
-//===-- WebAssemblyStoreResults.cpp - Optimize using store result values --===//
+//== WebAssemblyMemIntrinsicResults.cpp - Optimize memory intrinsic results ==//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -8,19 +8,22 @@
 //===----------------------------------------------------------------------===//
 ///
 /// \file
-/// This file implements an optimization pass using store result values.
+/// This file implements an optimization pass using memory intrinsic results.
 ///
-/// WebAssembly's store instructions return the stored value. This is to enable
-/// an optimization wherein uses of the stored value can be replaced by uses of
-/// the store's result value, making the stored value register more likely to
-/// be single-use, thus more likely to be useful to register stackifying, and
-/// potentially also exposing the store to register stackifying. These both can
-/// reduce get_local/set_local traffic.
+/// Calls to memory intrinsics (memcpy, memmove, memset) return the destination
+/// address. They are in the form of
+///   %dst_new = call @memcpy %dst, %src, %len
+/// where %dst and %dst_new registers contain the same value.
 ///
-/// This pass also performs this optimization for memcpy, memmove, and memset
-/// calls, since the LLVM intrinsics for these return void so they can't use the
-/// returned attribute and consequently aren't handled by the OptimizeReturned
-/// pass.
+/// This is to enable an optimization wherein uses of the %dst register used in
+/// the parameter can be replaced by uses of the %dst_new register used in the
+/// result, making the %dst register more likely to be single-use, thus more
+/// likely to be useful to register stackifying, and potentially also exposing
+/// the call instruction itself to register stackifying. These both can reduce
+/// local.get/local.set traffic.
+///
+/// The LLVM intrinsics for these return void so they can't use the returned
+/// attribute and consequently aren't handled by the OptimizeReturned pass.
 ///
 //===----------------------------------------------------------------------===//
 
@@ -38,15 +41,17 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-#define DEBUG_TYPE "wasm-store-results"
+#define DEBUG_TYPE "wasm-mem-intrinsic-results"
 
 namespace {
-class WebAssemblyStoreResults final : public MachineFunctionPass {
+class WebAssemblyMemIntrinsicResults final : public MachineFunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid
-  WebAssemblyStoreResults() : MachineFunctionPass(ID) {}
+  WebAssemblyMemIntrinsicResults() : MachineFunctionPass(ID) {}
 
-  StringRef getPassName() const override { return "WebAssembly Store Results"; }
+  StringRef getPassName() const override {
+    return "WebAssembly Memory Intrinsic Results";
+  }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
@@ -67,12 +72,13 @@ private:
 };
 } // end anonymous namespace
 
-char WebAssemblyStoreResults::ID = 0;
-INITIALIZE_PASS(WebAssemblyStoreResults, DEBUG_TYPE,
-                "Optimize store result values for WebAssembly", false, false)
+char WebAssemblyMemIntrinsicResults::ID = 0;
+INITIALIZE_PASS(WebAssemblyMemIntrinsicResults, DEBUG_TYPE,
+                "Optimize memory intrinsic result values for WebAssembly",
+                false, false)
 
-FunctionPass *llvm::createWebAssemblyStoreResults() {
-  return new WebAssemblyStoreResults();
+FunctionPass *llvm::createWebAssemblyMemIntrinsicResults() {
+  return new WebAssemblyMemIntrinsicResults();
 }
 
 // Replace uses of FromReg with ToReg if they are dominated by MI.
@@ -91,7 +97,8 @@ static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI,
 
   SmallVector<SlotIndex, 4> Indices;
 
-  for (auto I = MRI.use_nodbg_begin(FromReg), E = MRI.use_nodbg_end(); I != E;) {
+  for (auto I = MRI.use_nodbg_begin(FromReg), E = MRI.use_nodbg_end();
+       I != E;) {
     MachineOperand &O = *I++;
     MachineInstr *Where = O.getParent();
 
@@ -132,9 +139,9 @@ static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI,
 
     // If we replaced all dominated uses, FromReg is now killed at MI.
     if (!FromLI->liveAt(FromIdx.getDeadSlot()))
-      MI.addRegisterKilled(FromReg,
-                           MBB.getParent()->getSubtarget<WebAssemblySubtarget>()
-                                 .getRegisterInfo());
+      MI.addRegisterKilled(FromReg, MBB.getParent()
+                                        ->getSubtarget<WebAssemblySubtarget>()
+                                        .getRegisterInfo());
   }
 
   return Changed;
@@ -142,8 +149,7 @@ static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI,
 
 static bool optimizeCall(MachineBasicBlock &MBB, MachineInstr &MI,
                          const MachineRegisterInfo &MRI,
-                         MachineDominatorTree &MDT,
-                         LiveIntervals &LIS,
+                         MachineDominatorTree &MDT, LiveIntervals &LIS,
                          const WebAssemblyTargetLowering &TLI,
                          const TargetLibraryInfo &LibInfo) {
   MachineOperand &Op1 = MI.getOperand(1);
@@ -164,14 +170,14 @@ static bool optimizeCall(MachineBasicBlock &MBB, MachineInstr &MI,
   unsigned FromReg = MI.getOperand(2).getReg();
   unsigned ToReg = MI.getOperand(0).getReg();
   if (MRI.getRegClass(FromReg) != MRI.getRegClass(ToReg))
-    report_fatal_error("Store results: call to builtin function with wrong "
-                       "signature, from/to mismatch");
+    report_fatal_error("Memory Intrinsic results: call to builtin function "
+                       "with wrong signature, from/to mismatch");
   return ReplaceDominatedUses(MBB, MI, FromReg, ToReg, MRI, MDT, LIS);
 }
 
-bool WebAssemblyStoreResults::runOnMachineFunction(MachineFunction &MF) {
+bool WebAssemblyMemIntrinsicResults::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG({
-    dbgs() << "********** Store Results **********\n"
+    dbgs() << "********** Memory Intrinsic Results **********\n"
            << "********** Function: " << MF.getName() << '\n';
   });
 
@@ -186,7 +192,8 @@ bool WebAssemblyStoreResults::runOnMachineFunction(MachineFunction &MF) {
   // We don't preserve SSA form.
   MRI.leaveSSA();
 
-  assert(MRI.tracksLiveness() && "StoreResults expects liveness tracking");
+  assert(MRI.tracksLiveness() &&
+         "MemIntrinsicResults expects liveness tracking");
 
   for (auto &MBB : MF) {
     LLVM_DEBUG(dbgs() << "Basic Block: " << MBB.getName() << '\n');
diff --git a/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp b/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
index 04ac22a589ea..3d0a15244ee0 100644
--- a/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
@@ -65,7 +65,8 @@ FunctionPass *llvm::createWebAssemblyOptimizeLiveIntervals() {
   return new WebAssemblyOptimizeLiveIntervals();
 }
 
-bool WebAssemblyOptimizeLiveIntervals::runOnMachineFunction(MachineFunction &MF) {
+bool WebAssemblyOptimizeLiveIntervals::runOnMachineFunction(
+    MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "********** Optimize LiveIntervals **********\n"
                        "********** Function: "
                     << MF.getName() << '\n');
@@ -76,11 +77,10 @@ bool WebAssemblyOptimizeLiveIntervals::runOnMachineFunction(MachineFunction &MF)
   // We don't preserve SSA form.
   MRI.leaveSSA();
 
-  assert(MRI.tracksLiveness() &&
-         "OptimizeLiveIntervals expects liveness");
+  assert(MRI.tracksLiveness() && "OptimizeLiveIntervals expects liveness");
 
   // Split multiple-VN LiveIntervals into multiple LiveIntervals.
-  SmallVector<LiveInterval*, 4> SplitLIs;
+  SmallVector<LiveInterval *, 4> SplitLIs;
   for (unsigned i = 0, e = MRI.getNumVirtRegs(); i < e; ++i) {
     unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
     if (MRI.reg_nodbg_empty(Reg))
@@ -94,7 +94,7 @@ bool WebAssemblyOptimizeLiveIntervals::runOnMachineFunction(MachineFunction &MF)
   // instructions to satisfy LiveIntervals' requirement that all uses be
   // dominated by defs. Now that LiveIntervals has computed which of these
   // defs are actually needed and which are dead, remove the dead ones.
-  for (auto MII = MF.begin()->begin(), MIE = MF.begin()->end(); MII != MIE; ) {
+  for (auto MII = MF.begin()->begin(), MIE = MF.begin()->end(); MII != MIE;) {
     MachineInstr *MI = &*MII++;
     if (MI->isImplicitDef() && MI->getOperand(0).isDead()) {
       LiveInterval &LI = LIS.getInterval(MI->getOperand(0).getReg());
diff --git a/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp b/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
index 113ee2532bce..2c018d0785a7 100644
--- a/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
@@ -74,6 +74,10 @@ void OptimizeReturned::visitCallSite(CallSite CS) {
 }
 
 bool OptimizeReturned::runOnFunction(Function &F) {
+  LLVM_DEBUG(dbgs() << "********** Optimize returned Attributes **********\n"
+                       "********** Function: "
+                    << F.getName() << '\n');
+
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   visit(F);
   return true;
diff --git a/lib/Target/WebAssembly/WebAssemblyPeephole.cpp b/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
index a54484407805..2dfd85953f14 100644
--- a/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
@@ -192,11 +192,21 @@ bool WebAssemblyPeephole::runOnMachineFunction(MachineFunction &MF) {
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v4i32,
             WebAssembly::COPY_V128);
         break;
+      case WebAssembly::RETURN_v2i64:
+        Changed |= MaybeRewriteToFallthrough(
+            MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v2i64,
+            WebAssembly::COPY_V128);
+        break;
       case WebAssembly::RETURN_v4f32:
         Changed |= MaybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v4f32,
             WebAssembly::COPY_V128);
         break;
+      case WebAssembly::RETURN_v2f64:
+        Changed |= MaybeRewriteToFallthrough(
+            MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v2f64,
+            WebAssembly::COPY_V128);
+        break;
       case WebAssembly::RETURN_VOID:
         Changed |= MaybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_VOID,
diff --git a/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp b/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
index e44e7057e233..0be0ba657830 100644
--- a/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
@@ -70,7 +70,8 @@ static bool HasArgumentDef(unsigned Reg, const MachineRegisterInfo &MRI) {
   return false;
 }
 
-bool WebAssemblyPrepareForLiveIntervals::runOnMachineFunction(MachineFunction &MF) {
+bool WebAssemblyPrepareForLiveIntervals::runOnMachineFunction(
+    MachineFunction &MF) {
   LLVM_DEBUG({
     dbgs() << "********** Prepare For LiveIntervals **********\n"
            << "********** Function: " << MF.getName() << '\n';
@@ -112,7 +113,7 @@ bool WebAssemblyPrepareForLiveIntervals::runOnMachineFunction(MachineFunction &M
 
   // Move ARGUMENT_* instructions to the top of the entry block, so that their
   // liveness reflects the fact that these really are live-in values.
-  for (auto MII = Entry.begin(), MIE = Entry.end(); MII != MIE; ) {
+  for (auto MII = Entry.begin(), MIE = Entry.end(); MII != MIE;) {
     MachineInstr &MI = *MII++;
     if (WebAssembly::isArgument(MI)) {
       MI.removeFromParent();
diff --git a/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp b/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
index d69a27937105..d97b13a8d699 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
@@ -118,16 +118,15 @@ bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
   // registers), by weight next, and then by position.
   // TODO: Investigate more intelligent sorting heuristics. For starters, we
   // should try to coalesce adjacent live intervals before non-adjacent ones.
-  llvm::sort(SortedIntervals.begin(), SortedIntervals.end(),
-             [MRI](LiveInterval *LHS, LiveInterval *RHS) {
-               if (MRI->isLiveIn(LHS->reg) != MRI->isLiveIn(RHS->reg))
-                 return MRI->isLiveIn(LHS->reg);
-               if (LHS->weight != RHS->weight)
-                 return LHS->weight > RHS->weight;
-               if (LHS->empty() || RHS->empty())
-                 return !LHS->empty() && RHS->empty();
-               return *LHS < *RHS;
-             });
+  llvm::sort(SortedIntervals, [MRI](LiveInterval *LHS, LiveInterval *RHS) {
+    if (MRI->isLiveIn(LHS->reg) != MRI->isLiveIn(RHS->reg))
+      return MRI->isLiveIn(LHS->reg);
+    if (LHS->weight != RHS->weight)
+      return LHS->weight > RHS->weight;
+    if (LHS->empty() || RHS->empty())
+      return !LHS->empty() && RHS->empty();
+    return *LHS < *RHS;
+  });
 
   LLVM_DEBUG(dbgs() << "Coloring register intervals:\n");
   SmallVector<unsigned, 16> SlotMapping(SortedIntervals.size(), -1u);
diff --git a/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp b/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
index 9f5d5bd87831..1eb32ed64494 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
@@ -22,9 +22,11 @@
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" // for WebAssembly::ARGUMENT_*
 #include "WebAssembly.h"
+#include "WebAssemblyDebugValueManager.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
@@ -97,11 +99,11 @@ static void ImposeStackOrdering(MachineInstr *MI) {
 static void ConvertImplicitDefToConstZero(MachineInstr *MI,
                                           MachineRegisterInfo &MRI,
                                           const TargetInstrInfo *TII,
-                                          MachineFunction &MF) {
+                                          MachineFunction &MF,
+                                          LiveIntervals &LIS) {
   assert(MI->getOpcode() == TargetOpcode::IMPLICIT_DEF);
 
-  const auto *RegClass =
-      MRI.getRegClass(MI->getOperand(0).getReg());
+  const auto *RegClass = MRI.getRegClass(MI->getOperand(0).getReg());
   if (RegClass == &WebAssembly::I32RegClass) {
     MI->setDesc(TII->get(WebAssembly::CONST_I32));
     MI->addOperand(MachineOperand::CreateImm(0));
@@ -118,6 +120,14 @@ static void ConvertImplicitDefToConstZero(MachineInstr *MI,
     ConstantFP *Val = cast<ConstantFP>(Constant::getNullValue(
         Type::getDoubleTy(MF.getFunction().getContext())));
     MI->addOperand(MachineOperand::CreateFPImm(Val));
+  } else if (RegClass == &WebAssembly::V128RegClass) {
+    unsigned TempReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+    MI->setDesc(TII->get(WebAssembly::SPLAT_v4i32));
+    MI->addOperand(MachineOperand::CreateReg(TempReg, false));
+    MachineInstr *Const = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+                                  TII->get(WebAssembly::CONST_I32), TempReg)
+                              .addImm(0);
+    LIS.InsertMachineInstrInMaps(*Const);
   } else {
     llvm_unreachable("Unexpected reg class");
   }
@@ -172,29 +182,24 @@ static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
   // Check for stores.
   if (MI.mayStore()) {
     Write = true;
-
-    // Check for stores to __stack_pointer.
-    for (auto MMO : MI.memoperands()) {
-      const MachinePointerInfo &MPI = MMO->getPointerInfo();
-      if (MPI.V.is<const PseudoSourceValue *>()) {
-        auto PSV = MPI.V.get<const PseudoSourceValue *>();
-        if (const ExternalSymbolPseudoSourceValue *EPSV =
-                dyn_cast<ExternalSymbolPseudoSourceValue>(PSV))
-          if (StringRef(EPSV->getSymbol()) == "__stack_pointer") {
-            StackPointer = true;
-          }
-      }
-    }
   } else if (MI.hasOrderedMemoryRef()) {
     switch (MI.getOpcode()) {
-    case WebAssembly::DIV_S_I32: case WebAssembly::DIV_S_I64:
-    case WebAssembly::REM_S_I32: case WebAssembly::REM_S_I64:
-    case WebAssembly::DIV_U_I32: case WebAssembly::DIV_U_I64:
-    case WebAssembly::REM_U_I32: case WebAssembly::REM_U_I64:
-    case WebAssembly::I32_TRUNC_S_F32: case WebAssembly::I64_TRUNC_S_F32:
-    case WebAssembly::I32_TRUNC_S_F64: case WebAssembly::I64_TRUNC_S_F64:
-    case WebAssembly::I32_TRUNC_U_F32: case WebAssembly::I64_TRUNC_U_F32:
-    case WebAssembly::I32_TRUNC_U_F64: case WebAssembly::I64_TRUNC_U_F64:
+    case WebAssembly::DIV_S_I32:
+    case WebAssembly::DIV_S_I64:
+    case WebAssembly::REM_S_I32:
+    case WebAssembly::REM_S_I64:
+    case WebAssembly::DIV_U_I32:
+    case WebAssembly::DIV_U_I64:
+    case WebAssembly::REM_U_I32:
+    case WebAssembly::REM_U_I64:
+    case WebAssembly::I32_TRUNC_S_F32:
+    case WebAssembly::I64_TRUNC_S_F32:
+    case WebAssembly::I32_TRUNC_S_F64:
+    case WebAssembly::I64_TRUNC_S_F64:
+    case WebAssembly::I32_TRUNC_U_F32:
+    case WebAssembly::I64_TRUNC_U_F32:
+    case WebAssembly::I32_TRUNC_U_F64:
+    case WebAssembly::I64_TRUNC_U_F64:
       // These instruction have hasUnmodeledSideEffects() returning true
       // because they trap on overflow and invalid so they can't be arbitrarily
       // moved, however hasOrderedMemoryRef() interprets this plus their lack
@@ -214,14 +219,22 @@ static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
   // Check for side effects.
   if (MI.hasUnmodeledSideEffects()) {
     switch (MI.getOpcode()) {
-    case WebAssembly::DIV_S_I32: case WebAssembly::DIV_S_I64:
-    case WebAssembly::REM_S_I32: case WebAssembly::REM_S_I64:
-    case WebAssembly::DIV_U_I32: case WebAssembly::DIV_U_I64:
-    case WebAssembly::REM_U_I32: case WebAssembly::REM_U_I64:
-    case WebAssembly::I32_TRUNC_S_F32: case WebAssembly::I64_TRUNC_S_F32:
-    case WebAssembly::I32_TRUNC_S_F64: case WebAssembly::I64_TRUNC_S_F64:
-    case WebAssembly::I32_TRUNC_U_F32: case WebAssembly::I64_TRUNC_U_F32:
-    case WebAssembly::I32_TRUNC_U_F64: case WebAssembly::I64_TRUNC_U_F64:
+    case WebAssembly::DIV_S_I32:
+    case WebAssembly::DIV_S_I64:
+    case WebAssembly::REM_S_I32:
+    case WebAssembly::REM_S_I64:
+    case WebAssembly::DIV_U_I32:
+    case WebAssembly::DIV_U_I64:
+    case WebAssembly::REM_U_I32:
+    case WebAssembly::REM_U_I64:
+    case WebAssembly::I32_TRUNC_S_F32:
+    case WebAssembly::I64_TRUNC_S_F32:
+    case WebAssembly::I32_TRUNC_S_F64:
+    case WebAssembly::I64_TRUNC_S_F64:
+    case WebAssembly::I32_TRUNC_U_F32:
+    case WebAssembly::I64_TRUNC_U_F32:
+    case WebAssembly::I32_TRUNC_U_F64:
+    case WebAssembly::I64_TRUNC_U_F64:
       // These instructions have hasUnmodeledSideEffects() returning true
       // because they trap on overflow and invalid so they can't be arbitrarily
       // moved, however in the specific case of register stackifying, it is safe
@@ -233,22 +246,15 @@ static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
     }
   }
 
+  // Check for writes to __stack_pointer global.
+  if (MI.getOpcode() == WebAssembly::GLOBAL_SET_I32 &&
+      strcmp(MI.getOperand(0).getSymbolName(), "__stack_pointer") == 0)
+    StackPointer = true;
+
   // Analyze calls.
   if (MI.isCall()) {
-    switch (MI.getOpcode()) {
-    case WebAssembly::CALL_VOID:
-    case WebAssembly::CALL_INDIRECT_VOID:
-      QueryCallee(MI, 0, Read, Write, Effects, StackPointer);
-      break;
-    case WebAssembly::CALL_I32: case WebAssembly::CALL_I64:
-    case WebAssembly::CALL_F32: case WebAssembly::CALL_F64:
-    case WebAssembly::CALL_INDIRECT_I32: case WebAssembly::CALL_INDIRECT_I64:
-    case WebAssembly::CALL_INDIRECT_F32: case WebAssembly::CALL_INDIRECT_F64:
-      QueryCallee(MI, 1, Read, Write, Effects, StackPointer);
-      break;
-    default:
-      llvm_unreachable("unexpected call opcode");
-    }
+    unsigned CalleeOpNo = WebAssembly::getCalleeOpNo(MI);
+    QueryCallee(MI, CalleeOpNo, Read, Write, Effects, StackPointer);
   }
 }
 
@@ -263,8 +269,7 @@ static bool ShouldRematerialize(const MachineInstr &Def, AliasAnalysis &AA,
 // LiveIntervals to handle complex cases.
 static MachineInstr *GetVRegDef(unsigned Reg, const MachineInstr *Insert,
                                 const MachineRegisterInfo &MRI,
-                                const LiveIntervals &LIS)
-{
+                                const LiveIntervals &LIS) {
   // Most registers are in SSA form here so we try a quick MRI query first.
   if (MachineInstr *Def = MRI.getUniqueVRegDef(Reg))
     return Def;
@@ -280,17 +285,16 @@ static MachineInstr *GetVRegDef(unsigned Reg, const MachineInstr *Insert,
 // Test whether Reg, as defined at Def, has exactly one use. This is a
 // generalization of MachineRegisterInfo::hasOneUse that uses LiveIntervals
 // to handle complex cases.
-static bool HasOneUse(unsigned Reg, MachineInstr *Def,
-                      MachineRegisterInfo &MRI, MachineDominatorTree &MDT,
-                      LiveIntervals &LIS) {
+static bool HasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI,
+                      MachineDominatorTree &MDT, LiveIntervals &LIS) {
   // Most registers are in SSA form here so we try a quick MRI query first.
   if (MRI.hasOneUse(Reg))
     return true;
 
   bool HasOne = false;
   const LiveInterval &LI = LIS.getInterval(Reg);
-  const VNInfo *DefVNI = LI.getVNInfoAt(
-      LIS.getInstructionIndex(*Def).getRegSlot());
+  const VNInfo *DefVNI =
+      LI.getVNInfoAt(LIS.getInstructionIndex(*Def).getRegSlot());
   assert(DefVNI);
   for (auto &I : MRI.use_nodbg_operands(Reg)) {
     const auto &Result = LI.Query(LIS.getInstructionIndex(*I.getParent()));
@@ -403,7 +407,6 @@ static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
     if (UseVNI != OneUseVNI)
       continue;
 
-    const MachineInstr *OneUseInst = OneUse.getParent();
     if (UseInst == OneUseInst) {
       // Another use in the same instruction. We need to ensure that the one
       // selected use happens "before" it.
@@ -415,8 +418,8 @@ static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
         // Actually, dominating is over-conservative. Test that the use would
         // happen after the one selected use in the stack evaluation order.
         //
-        // This is needed as a consequence of using implicit get_locals for
-        // uses and implicit set_locals for defs.
+        // This is needed as a consequence of using implicit local.gets for
+        // uses and implicit local.sets for defs.
         if (UseInst->getDesc().getNumDefs() == 0)
           return false;
         const MachineOperand &MO = UseInst->getOperand(0);
@@ -426,8 +429,8 @@ static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
         if (!TargetRegisterInfo::isVirtualRegister(DefReg) ||
             !MFI.isVRegStackified(DefReg))
           return false;
-        assert(MRI.hasOneUse(DefReg));
-        const MachineOperand &NewUse = *MRI.use_begin(DefReg);
+        assert(MRI.hasOneNonDBGUse(DefReg));
+        const MachineOperand &NewUse = *MRI.use_nodbg_begin(DefReg);
         const MachineInstr *NewUseInst = NewUse.getParent();
         if (NewUseInst == OneUseInst) {
           if (&OneUse > &NewUse)
@@ -459,22 +462,23 @@ static unsigned GetTeeOpcode(const TargetRegisterClass *RC) {
 // Shrink LI to its uses, cleaning up LI.
 static void ShrinkToUses(LiveInterval &LI, LiveIntervals &LIS) {
   if (LIS.shrinkToUses(&LI)) {
-    SmallVector<LiveInterval*, 4> SplitLIs;
+    SmallVector<LiveInterval *, 4> SplitLIs;
     LIS.splitSeparateComponents(LI, SplitLIs);
   }
 }
 
 /// A single-use def in the same block with no intervening memory or register
 /// dependencies; move the def down and nest it with the current instruction.
-static MachineInstr *MoveForSingleUse(unsigned Reg, MachineOperand& Op,
-                                      MachineInstr *Def,
-                                      MachineBasicBlock &MBB,
+static MachineInstr *MoveForSingleUse(unsigned Reg, MachineOperand &Op,
+                                      MachineInstr *Def, MachineBasicBlock &MBB,
                                       MachineInstr *Insert, LiveIntervals &LIS,
                                       WebAssemblyFunctionInfo &MFI,
                                       MachineRegisterInfo &MRI) {
   LLVM_DEBUG(dbgs() << "Move for single use: "; Def->dump());
 
+  WebAssemblyDebugValueManager DefDIs(Def);
   MBB.splice(Insert, &MBB, Def);
+  DefDIs.move(Insert);
   LIS.handleMove(*Def);
 
   if (MRI.hasOneDef(Reg) && MRI.hasOneUse(Reg)) {
@@ -499,6 +503,8 @@ static MachineInstr *MoveForSingleUse(unsigned Reg, MachineOperand& Op,
 
     MFI.stackifyVReg(NewReg);
 
+    DefDIs.updateReg(NewReg);
+
     LLVM_DEBUG(dbgs() << " - Replaced register: "; Def->dump());
   }
 
@@ -516,6 +522,8 @@ static MachineInstr *RematerializeCheapDef(
   LLVM_DEBUG(dbgs() << "Rematerializing cheap def: "; Def.dump());
   LLVM_DEBUG(dbgs() << " - for use in "; Op.getParent()->dump());
 
+  WebAssemblyDebugValueManager DefDIs(&Def);
+
   unsigned NewReg = MRI.createVirtualRegister(MRI.getRegClass(Reg));
   TII->reMaterialize(MBB, Insert, NewReg, 0, Def, *TRI);
   Op.setReg(NewReg);
@@ -536,6 +544,7 @@ static MachineInstr *RematerializeCheapDef(
   }
 
   // If that was the last use of the original, delete the original.
+  // Move or clone corresponding DBG_VALUEs to the 'Insert' location.
   if (IsDead) {
     LLVM_DEBUG(dbgs() << " - Deleting original\n");
     SlotIndex Idx = LIS.getInstructionIndex(Def).getRegSlot();
@@ -543,6 +552,11 @@ static MachineInstr *RematerializeCheapDef(
     LIS.removeInterval(Reg);
     LIS.RemoveMachineInstrFromMaps(Def);
     Def.eraseFromParent();
+
+    DefDIs.move(&*Insert);
+    DefDIs.updateReg(NewReg);
+  } else {
+    DefDIs.clone(&*Insert, NewReg);
   }
 
   return Clone;
@@ -566,7 +580,7 @@ static MachineInstr *RematerializeCheapDef(
 ///    INST ..., Reg, ...
 ///    INST ..., Reg, ...
 ///
-/// with DefReg and TeeReg stackified. This eliminates a get_local from the
+/// with DefReg and TeeReg stackified. This eliminates a local.get from the
 /// resulting code.
 static MachineInstr *MoveAndTeeForMultiUse(
     unsigned Reg, MachineOperand &Op, MachineInstr *Def, MachineBasicBlock &MBB,
@@ -574,6 +588,8 @@ static MachineInstr *MoveAndTeeForMultiUse(
     MachineRegisterInfo &MRI, const WebAssemblyInstrInfo *TII) {
   LLVM_DEBUG(dbgs() << "Move and tee for multi-use:"; Def->dump());
 
+  WebAssemblyDebugValueManager DefDIs(Def);
+
   // Move Def into place.
   MBB.splice(Insert, &MBB, Def);
   LIS.handleMove(*Def);
@@ -592,6 +608,8 @@ static MachineInstr *MoveAndTeeForMultiUse(
   SlotIndex TeeIdx = LIS.InsertMachineInstrInMaps(*Tee).getRegSlot();
   SlotIndex DefIdx = LIS.getInstructionIndex(*Def).getRegSlot();
 
+  DefDIs.move(Insert);
+
   // Tell LiveIntervals we moved the original vreg def from Def to Tee.
   LiveInterval &LI = LIS.getInterval(Reg);
   LiveInterval::iterator I = LI.FindSegmentContaining(DefIdx);
@@ -608,6 +626,9 @@ static MachineInstr *MoveAndTeeForMultiUse(
   ImposeStackOrdering(Def);
   ImposeStackOrdering(Tee);
 
+  DefDIs.clone(Tee, DefReg);
+  DefDIs.clone(Insert, TeeReg);
+
   LLVM_DEBUG(dbgs() << " - Replaced register: "; Def->dump());
   LLVM_DEBUG(dbgs() << " - Tee instruction: "; Tee->dump());
   return Def;
@@ -672,8 +693,8 @@ public:
   /// operand in the tree that we haven't visited yet. Moving a definition of
   /// Reg to a point in the tree after that would change its value.
   ///
-  /// This is needed as a consequence of using implicit get_locals for
-  /// uses and implicit set_locals for defs.
+  /// This is needed as a consequence of using implicit local.gets for
+  /// uses and implicit local.sets for defs.
   bool IsOnStack(unsigned Reg) const {
     for (const RangeTy &Range : Worklist)
       for (const MachineOperand &MO : Range)
@@ -687,9 +708,9 @@ public:
 /// tried for the current instruction and didn't work.
 class CommutingState {
   /// There are effectively three states: the initial state where we haven't
-  /// started commuting anything and we don't know anything yet, the tenative
+  /// started commuting anything and we don't know anything yet, the tentative
   /// state where we've commuted the operands of the current instruction and are
-  /// revisting it, and the declined state where we've reverted the operands
+  /// revisiting it, and the declined state where we've reverted the operands
   /// back to their original order and will no longer commute it further.
   bool TentativelyCommuting;
   bool Declined;
@@ -831,7 +852,7 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
         // to a constant 0 so that the def is explicit, and the push/pop
         // correspondence is maintained.
         if (Insert->getOpcode() == TargetOpcode::IMPLICIT_DEF)
-          ConvertImplicitDefToConstZero(Insert, MRI, TII, MF);
+          ConvertImplicitDefToConstZero(Insert, MRI, TII, MF, LIS);
 
         // We stackified an operand. Add the defining instruction's operands to
         // the worklist stack now to continue to build an ever deeper tree.
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
index b6481ac2d4ae..1f0870865b06 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
@@ -22,9 +22,9 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/IR/Function.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/Target/TargetOptions.h"
 using namespace llvm;
 
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
index 29f42b96b249..a7c3d177724d 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
@@ -63,6 +63,6 @@ def I32 : WebAssemblyRegClass<[i32], 32, (add FP32, SP32, I32_0)>;
 def I64 : WebAssemblyRegClass<[i64], 64, (add FP64, SP64, I64_0)>;
 def F32 : WebAssemblyRegClass<[f32], 32, (add F32_0)>;
 def F64 : WebAssemblyRegClass<[f64], 64, (add F64_0)>;
-def V128 : WebAssemblyRegClass<[v4f32, v4i32, v16i8, v8i16], 128, (add V128_0)>;
+def V128 : WebAssemblyRegClass<[v4f32, v2f64, v2i64, v4i32, v16i8, v8i16], 128,
+                               (add V128_0)>;
 def EXCEPT_REF : WebAssemblyRegClass<[ExceptRef], 0, (add EXCEPT_REF_0)>;
-
diff --git a/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp b/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
index f432b367d156..e5a3e47a3bcd 100644
--- a/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
@@ -54,8 +54,8 @@ private:
 
 char WebAssemblyReplacePhysRegs::ID = 0;
 INITIALIZE_PASS(WebAssemblyReplacePhysRegs, DEBUG_TYPE,
-                "Replace physical registers with virtual registers",
-                false, false)
+                "Replace physical registers with virtual registers", false,
+                false)
 
 FunctionPass *llvm::createWebAssemblyReplacePhysRegs() {
   return new WebAssemblyReplacePhysRegs();
@@ -86,7 +86,7 @@ bool WebAssemblyReplacePhysRegs::runOnMachineFunction(MachineFunction &MF) {
     // Replace explicit uses of the physical register with a virtual register.
     const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(PReg);
     unsigned VReg = WebAssembly::NoRegister;
-    for (auto I = MRI.reg_begin(PReg), E = MRI.reg_end(); I != E; ) {
+    for (auto I = MRI.reg_begin(PReg), E = MRI.reg_end(); I != E;) {
       MachineOperand &MO = *I++;
       if (!MO.isImplicit()) {
         if (VReg == WebAssembly::NoRegister)
diff --git a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index fe8a5e4c06f1..6cf81a9d77b3 100644
--- a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -88,7 +88,6 @@ enum RuntimeLibcallSignature {
   unsupported
 };
 
-
 struct RuntimeLibcallSignatureTable {
   std::vector<RuntimeLibcallSignature> Table;
 
@@ -486,18 +485,17 @@ struct StaticLibcallNameMap {
 
 } // end anonymous namespace
 
-
-
-void llvm::GetSignature(const WebAssemblySubtarget &Subtarget,
-                        RTLIB::Libcall LC, SmallVectorImpl<wasm::ValType> &Rets,
-                        SmallVectorImpl<wasm::ValType> &Params) {
+void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+                               RTLIB::Libcall LC,
+                               SmallVectorImpl<wasm::ValType> &Rets,
+                               SmallVectorImpl<wasm::ValType> &Params) {
   assert(Rets.empty());
   assert(Params.empty());
 
   wasm::ValType iPTR =
       Subtarget.hasAddr64() ? wasm::ValType::I64 : wasm::ValType::I32;
 
-  auto& Table = RuntimeLibcallSignatures->Table;
+  auto &Table = RuntimeLibcallSignatures->Table;
   switch (Table[LC]) {
   case func:
     break;
@@ -834,11 +832,12 @@ void llvm::GetSignature(const WebAssemblySubtarget &Subtarget,
 static ManagedStatic<StaticLibcallNameMap> LibcallNameMap;
 // TODO: If the RTLIB::Libcall-taking flavor of GetSignature remains unsed
 // other than here, just roll its logic into this version.
-void llvm::GetSignature(const WebAssemblySubtarget &Subtarget, const char *Name,
-                        SmallVectorImpl<wasm::ValType> &Rets,
-                        SmallVectorImpl<wasm::ValType> &Params) {
-  auto& Map = LibcallNameMap->Map;
+void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+                               const char *Name,
+                               SmallVectorImpl<wasm::ValType> &Rets,
+                               SmallVectorImpl<wasm::ValType> &Params) {
+  auto &Map = LibcallNameMap->Map;
   auto val = Map.find(Name);
   assert(val != Map.end() && "unexpected runtime library name");
-  return GetSignature(Subtarget, val->second, Rets, Params);
+  return GetLibcallSignature(Subtarget, val->second, Rets, Params);
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
index 2ba65ff5b716..7fa70bea96de 100644
--- a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
+++ b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
@@ -23,14 +23,15 @@ namespace llvm {
 
 class WebAssemblySubtarget;
 
-extern void GetSignature(const WebAssemblySubtarget &Subtarget,
-                         RTLIB::Libcall LC,
-                         SmallVectorImpl<wasm::ValType> &Rets,
-                         SmallVectorImpl<wasm::ValType> &Params);
+extern void GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+                                RTLIB::Libcall LC,
+                                SmallVectorImpl<wasm::ValType> &Rets,
+                                SmallVectorImpl<wasm::ValType> &Params);
 
-extern void GetSignature(const WebAssemblySubtarget &Subtarget,
-                         const char *Name, SmallVectorImpl<wasm::ValType> &Rets,
-                         SmallVectorImpl<wasm::ValType> &Params);
+extern void GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+                                const char *Name,
+                                SmallVectorImpl<wasm::ValType> &Rets,
+                                SmallVectorImpl<wasm::ValType> &Params);
 
 } // end namespace llvm
 
diff --git a/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp b/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
index 14221993603a..c95af88c6f43 100644
--- a/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
+++ b/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
@@ -60,8 +60,7 @@ static void RewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
   assert(MI.hasOneMemOperand() &&
          "Load and store instructions have exactly one mem operand");
   assert((*MI.memoperands_begin())->getSize() ==
-             (UINT64_C(1)
-              << WebAssembly::GetDefaultP2Align(MI.getOpcode())) &&
+             (UINT64_C(1) << WebAssembly::GetDefaultP2Align(MI.getOpcode())) &&
          "Default p2align value should be natural");
   assert(MI.getDesc().OpInfo[OperandNo].OperandType ==
              WebAssembly::OPERAND_P2ALIGN &&
@@ -69,8 +68,8 @@ static void RewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
   uint64_t P2Align = Log2_64((*MI.memoperands_begin())->getAlignment());
 
   // WebAssembly does not currently support supernatural alignment.
-  P2Align = std::min(
-      P2Align, uint64_t(WebAssembly::GetDefaultP2Align(MI.getOpcode())));
+  P2Align = std::min(P2Align,
+                     uint64_t(WebAssembly::GetDefaultP2Align(MI.getOpcode())));
 
   MI.getOperand(OperandNo).setImm(P2Align);
 }
@@ -90,6 +89,12 @@ bool WebAssemblySetP2AlignOperands::runOnMachineFunction(MachineFunction &MF) {
       case WebAssembly::LOAD_I64:
       case WebAssembly::LOAD_F32:
       case WebAssembly::LOAD_F64:
+      case WebAssembly::LOAD_v16i8:
+      case WebAssembly::LOAD_v8i16:
+      case WebAssembly::LOAD_v4i32:
+      case WebAssembly::LOAD_v2i64:
+      case WebAssembly::LOAD_v4f32:
+      case WebAssembly::LOAD_v2f64:
       case WebAssembly::LOAD8_S_I32:
       case WebAssembly::LOAD8_U_I32:
       case WebAssembly::LOAD16_S_I32:
@@ -119,6 +124,8 @@ bool WebAssemblySetP2AlignOperands::runOnMachineFunction(MachineFunction &MF) {
       case WebAssembly::ATOMIC_RMW8_U_XOR_I64:
       case WebAssembly::ATOMIC_RMW8_U_XCHG_I32:
       case WebAssembly::ATOMIC_RMW8_U_XCHG_I64:
+      case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I32:
+      case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I64:
       case WebAssembly::ATOMIC_RMW16_U_ADD_I32:
       case WebAssembly::ATOMIC_RMW16_U_ADD_I64:
       case WebAssembly::ATOMIC_RMW16_U_SUB_I32:
@@ -131,6 +138,8 @@ bool WebAssemblySetP2AlignOperands::runOnMachineFunction(MachineFunction &MF) {
       case WebAssembly::ATOMIC_RMW16_U_XOR_I64:
       case WebAssembly::ATOMIC_RMW16_U_XCHG_I32:
       case WebAssembly::ATOMIC_RMW16_U_XCHG_I64:
+      case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I32:
+      case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I64:
       case WebAssembly::ATOMIC_RMW_ADD_I32:
       case WebAssembly::ATOMIC_RMW32_U_ADD_I64:
       case WebAssembly::ATOMIC_RMW_SUB_I32:
@@ -143,18 +152,30 @@ bool WebAssemblySetP2AlignOperands::runOnMachineFunction(MachineFunction &MF) {
       case WebAssembly::ATOMIC_RMW32_U_XOR_I64:
       case WebAssembly::ATOMIC_RMW_XCHG_I32:
       case WebAssembly::ATOMIC_RMW32_U_XCHG_I64:
+      case WebAssembly::ATOMIC_RMW_CMPXCHG_I32:
+      case WebAssembly::ATOMIC_RMW32_U_CMPXCHG_I64:
       case WebAssembly::ATOMIC_RMW_ADD_I64:
       case WebAssembly::ATOMIC_RMW_SUB_I64:
       case WebAssembly::ATOMIC_RMW_AND_I64:
       case WebAssembly::ATOMIC_RMW_OR_I64:
       case WebAssembly::ATOMIC_RMW_XOR_I64:
       case WebAssembly::ATOMIC_RMW_XCHG_I64:
+      case WebAssembly::ATOMIC_RMW_CMPXCHG_I64:
+      case WebAssembly::ATOMIC_NOTIFY:
+      case WebAssembly::ATOMIC_WAIT_I32:
+      case WebAssembly::ATOMIC_WAIT_I64:
         RewriteP2Align(MI, WebAssembly::LoadP2AlignOperandNo);
         break;
       case WebAssembly::STORE_I32:
       case WebAssembly::STORE_I64:
       case WebAssembly::STORE_F32:
       case WebAssembly::STORE_F64:
+      case WebAssembly::STORE_v16i8:
+      case WebAssembly::STORE_v8i16:
+      case WebAssembly::STORE_v4i32:
+      case WebAssembly::STORE_v2i64:
+      case WebAssembly::STORE_v4f32:
+      case WebAssembly::STORE_v2f64:
       case WebAssembly::STORE8_I32:
       case WebAssembly::STORE16_I32:
       case WebAssembly::STORE8_I64:
diff --git a/lib/Target/WebAssembly/WebAssemblySubtarget.cpp b/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
index d6af0fb219d7..98133e2153a0 100644
--- a/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
+++ b/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
@@ -40,10 +40,9 @@ WebAssemblySubtarget::WebAssemblySubtarget(const Triple &TT,
                                            const std::string &CPU,
                                            const std::string &FS,
                                            const TargetMachine &TM)
-    : WebAssemblyGenSubtargetInfo(TT, CPU, FS), HasSIMD128(false),
-      HasAtomics(false), HasNontrappingFPToInt(false), HasSignExt(false),
-      HasExceptionHandling(false), CPUString(CPU), TargetTriple(TT),
-      FrameLowering(), InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(),
+    : WebAssemblyGenSubtargetInfo(TT, CPU, FS), CPUString(CPU),
+      TargetTriple(TT), FrameLowering(),
+      InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(),
       TLInfo(TM, *this) {}
 
 bool WebAssemblySubtarget::enableMachineScheduler() const {
diff --git a/lib/Target/WebAssembly/WebAssemblySubtarget.h b/lib/Target/WebAssembly/WebAssemblySubtarget.h
index b170dbff3b32..0a0c04609ac4 100644
--- a/lib/Target/WebAssembly/WebAssemblySubtarget.h
+++ b/lib/Target/WebAssembly/WebAssemblySubtarget.h
@@ -29,11 +29,16 @@
 namespace llvm {
 
 class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
-  bool HasSIMD128;
-  bool HasAtomics;
-  bool HasNontrappingFPToInt;
-  bool HasSignExt;
-  bool HasExceptionHandling;
+  enum SIMDEnum {
+    NoSIMD,
+    SIMD128,
+    UnimplementedSIMD128,
+  } SIMDLevel = NoSIMD;
+
+  bool HasAtomics = false;
+  bool HasNontrappingFPToInt = false;
+  bool HasSignExt = false;
+  bool HasExceptionHandling = false;
 
   /// String name of used CPU.
   std::string CPUString;
@@ -77,7 +82,10 @@ public:
 
   // Predicates used by WebAssemblyInstrInfo.td.
   bool hasAddr64() const { return TargetTriple.isArch64Bit(); }
-  bool hasSIMD128() const { return HasSIMD128; }
+  bool hasSIMD128() const { return SIMDLevel >= SIMD128; }
+  bool hasUnimplementedSIMD128() const {
+    return SIMDLevel >= UnimplementedSIMD128;
+  }
   bool hasAtomics() const { return HasAtomics; }
   bool hasNontrappingFPToInt() const { return HasNontrappingFPToInt; }
   bool hasSignExt() const { return HasSignExt; }
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 7c10f022cbbc..3bf8dd40892c 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -58,10 +58,11 @@ extern "C" void LLVMInitializeWebAssemblyTarget() {
   initializeOptimizeReturnedPass(PR);
   initializeWebAssemblyArgumentMovePass(PR);
   initializeWebAssemblySetP2AlignOperandsPass(PR);
+  initializeWebAssemblyEHRestoreStackPointerPass(PR);
   initializeWebAssemblyReplacePhysRegsPass(PR);
   initializeWebAssemblyPrepareForLiveIntervalsPass(PR);
   initializeWebAssemblyOptimizeLiveIntervalsPass(PR);
-  initializeWebAssemblyStoreResultsPass(PR);
+  initializeWebAssemblyMemIntrinsicResultsPass(PR);
   initializeWebAssemblyRegStackifyPass(PR);
   initializeWebAssemblyRegColoringPass(PR);
   initializeWebAssemblyExplicitLocalsPass(PR);
@@ -81,8 +82,12 @@ extern "C" void LLVMInitializeWebAssemblyTarget() {
 //===----------------------------------------------------------------------===//
 
 static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
-  if (!RM.hasValue())
-    return Reloc::PIC_;
+  if (!RM.hasValue()) {
+    // Default to static relocation model.  This should always be more optimial
+    // than PIC since the static linker can determine all global addresses and
+    // assume direct function calls.
+    return Reloc::Static;
+  }
   return *RM;
 }
 
@@ -96,7 +101,7 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine(
                         TT.isArch64Bit() ? "e-m:e-p:64:64-i64:64-n32:64-S128"
                                          : "e-m:e-p:32:32-i64:64-n32:64-S128",
                         TT, CPU, FS, Options, getEffectiveRelocModel(RM),
-                        CM ? *CM : CodeModel::Large, OL),
+                        getEffectiveCodeModel(CM, CodeModel::Large), OL),
       TLOF(new WebAssemblyTargetObjectFile()) {
   // WebAssembly type-checks instructions, but a noreturn function with a return
   // type that doesn't match the context will cause a check failure. So we lower
@@ -149,7 +154,7 @@ class StripThreadLocal final : public ModulePass {
   // pass just converts all GlobalVariables to NotThreadLocal
   static char ID;
 
- public:
+public:
   StripThreadLocal() : ModulePass(ID) {}
   bool runOnModule(Module &M) override {
     for (auto &GV : M.globals())
@@ -280,6 +285,9 @@ void WebAssemblyPassConfig::addPostRegAlloc() {
 void WebAssemblyPassConfig::addPreEmitPass() {
   TargetPassConfig::addPreEmitPass();
 
+  // Restore __stack_pointer global after an exception is thrown.
+  addPass(createWebAssemblyEHRestoreStackPointer());
+
   // Now that we have a prologue and epilogue and all frame indices are
   // rewritten, eliminate SP and FP. This allows them to be stackified,
   // colored, and numbered with the rest of the registers.
@@ -290,6 +298,12 @@ void WebAssemblyPassConfig::addPreEmitPass() {
   // order of the arguments.
   addPass(createWebAssemblyCallIndirectFixup());
 
+  // Eliminate multiple-entry loops.
+  addPass(createWebAssemblyFixIrreducibleControlFlow());
+
+  // Do various transformations for exception handling.
+  addPass(createWebAssemblyLateEHPrepare());
+
   if (getOptLevel() != CodeGenOpt::None) {
     // LiveIntervals isn't commonly run this late. Re-establish preconditions.
     addPass(createWebAssemblyPrepareForLiveIntervals());
@@ -297,13 +311,14 @@ void WebAssemblyPassConfig::addPreEmitPass() {
     // Depend on LiveIntervals and perform some optimizations on it.
     addPass(createWebAssemblyOptimizeLiveIntervals());
 
-    // Prepare store instructions for register stackifying.
-    addPass(createWebAssemblyStoreResults());
+    // Prepare memory intrinsic calls for register stackifying.
+    addPass(createWebAssemblyMemIntrinsicResults());
 
     // Mark registers as representing wasm's value stack. This is a key
     // code-compression technique in WebAssembly. We run this pass (and
-    // StoreResults above) very late, so that it sees as much code as possible,
-    // including code emitted by PEI and expanded by late tail duplication.
+    // MemIntrinsicResults above) very late, so that it sees as much code as
+    // possible, including code emitted by PEI and expanded by late tail
+    // duplication.
     addPass(createWebAssemblyRegStackify());
 
     // Run the register coloring pass to reduce the total number of registers.
@@ -312,17 +327,9 @@ void WebAssemblyPassConfig::addPreEmitPass() {
     addPass(createWebAssemblyRegColoring());
   }
 
-  // Eliminate multiple-entry loops. Do this before inserting explicit get_local
-  // and set_local operators because we create a new variable that we want
-  // converted into a local.
-  addPass(createWebAssemblyFixIrreducibleControlFlow());
-
-  // Insert explicit get_local and set_local operators.
+  // Insert explicit local.get and local.set operators.
   addPass(createWebAssemblyExplicitLocals());
 
-  // Do various transformations for exception handling
-  addPass(createWebAssemblyLateEHPrepare());
-
   // Sort the blocks of the CFG into topological order, a prerequisite for
   // BLOCK and LOOP markers.
   addPass(createWebAssemblyCFGSort());
diff --git a/lib/Target/WebAssembly/WebAssemblyUtilities.cpp b/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
index 5944cea5abd1..ada6fb9a96d7 100644
--- a/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
@@ -27,14 +27,26 @@ const char *const WebAssembly::PersonalityWrapperFn =
 
 bool WebAssembly::isArgument(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
-  case WebAssembly::ARGUMENT_I32:
-  case WebAssembly::ARGUMENT_I64:
-  case WebAssembly::ARGUMENT_F32:
-  case WebAssembly::ARGUMENT_F64:
+  case WebAssembly::ARGUMENT_i32:
+  case WebAssembly::ARGUMENT_i32_S:
+  case WebAssembly::ARGUMENT_i64:
+  case WebAssembly::ARGUMENT_i64_S:
+  case WebAssembly::ARGUMENT_f32:
+  case WebAssembly::ARGUMENT_f32_S:
+  case WebAssembly::ARGUMENT_f64:
+  case WebAssembly::ARGUMENT_f64_S:
   case WebAssembly::ARGUMENT_v16i8:
+  case WebAssembly::ARGUMENT_v16i8_S:
   case WebAssembly::ARGUMENT_v8i16:
+  case WebAssembly::ARGUMENT_v8i16_S:
   case WebAssembly::ARGUMENT_v4i32:
+  case WebAssembly::ARGUMENT_v4i32_S:
+  case WebAssembly::ARGUMENT_v2i64:
+  case WebAssembly::ARGUMENT_v2i64_S:
   case WebAssembly::ARGUMENT_v4f32:
+  case WebAssembly::ARGUMENT_v4f32_S:
+  case WebAssembly::ARGUMENT_v2f64:
+  case WebAssembly::ARGUMENT_v2f64_S:
     return true;
   default:
     return false;
@@ -44,9 +56,15 @@ bool WebAssembly::isArgument(const MachineInstr &MI) {
 bool WebAssembly::isCopy(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::COPY_I32:
+  case WebAssembly::COPY_I32_S:
   case WebAssembly::COPY_I64:
+  case WebAssembly::COPY_I64_S:
   case WebAssembly::COPY_F32:
+  case WebAssembly::COPY_F32_S:
   case WebAssembly::COPY_F64:
+  case WebAssembly::COPY_F64_S:
+  case WebAssembly::COPY_V128:
+  case WebAssembly::COPY_V128_S:
     return true;
   default:
     return false;
@@ -56,9 +74,15 @@ bool WebAssembly::isCopy(const MachineInstr &MI) {
 bool WebAssembly::isTee(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::TEE_I32:
+  case WebAssembly::TEE_I32_S:
   case WebAssembly::TEE_I64:
+  case WebAssembly::TEE_I64_S:
   case WebAssembly::TEE_F32:
+  case WebAssembly::TEE_F32_S:
   case WebAssembly::TEE_F64:
+  case WebAssembly::TEE_F64_S:
+  case WebAssembly::TEE_V128:
+  case WebAssembly::TEE_V128_S:
     return true;
   default:
     return false;
@@ -81,15 +105,29 @@ bool WebAssembly::isChild(const MachineInstr &MI,
 bool WebAssembly::isCallDirect(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::CALL_VOID:
+  case WebAssembly::CALL_VOID_S:
   case WebAssembly::CALL_I32:
+  case WebAssembly::CALL_I32_S:
   case WebAssembly::CALL_I64:
+  case WebAssembly::CALL_I64_S:
   case WebAssembly::CALL_F32:
+  case WebAssembly::CALL_F32_S:
   case WebAssembly::CALL_F64:
+  case WebAssembly::CALL_F64_S:
   case WebAssembly::CALL_v16i8:
+  case WebAssembly::CALL_v16i8_S:
   case WebAssembly::CALL_v8i16:
+  case WebAssembly::CALL_v8i16_S:
   case WebAssembly::CALL_v4i32:
+  case WebAssembly::CALL_v4i32_S:
+  case WebAssembly::CALL_v2i64:
+  case WebAssembly::CALL_v2i64_S:
   case WebAssembly::CALL_v4f32:
+  case WebAssembly::CALL_v4f32_S:
+  case WebAssembly::CALL_v2f64:
+  case WebAssembly::CALL_v2f64_S:
   case WebAssembly::CALL_EXCEPT_REF:
+  case WebAssembly::CALL_EXCEPT_REF_S:
     return true;
   default:
     return false;
@@ -99,15 +137,29 @@ bool WebAssembly::isCallDirect(const MachineInstr &MI) {
 bool WebAssembly::isCallIndirect(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::CALL_INDIRECT_VOID:
+  case WebAssembly::CALL_INDIRECT_VOID_S:
   case WebAssembly::CALL_INDIRECT_I32:
+  case WebAssembly::CALL_INDIRECT_I32_S:
   case WebAssembly::CALL_INDIRECT_I64:
+  case WebAssembly::CALL_INDIRECT_I64_S:
   case WebAssembly::CALL_INDIRECT_F32:
+  case WebAssembly::CALL_INDIRECT_F32_S:
   case WebAssembly::CALL_INDIRECT_F64:
+  case WebAssembly::CALL_INDIRECT_F64_S:
   case WebAssembly::CALL_INDIRECT_v16i8:
+  case WebAssembly::CALL_INDIRECT_v16i8_S:
   case WebAssembly::CALL_INDIRECT_v8i16:
+  case WebAssembly::CALL_INDIRECT_v8i16_S:
   case WebAssembly::CALL_INDIRECT_v4i32:
+  case WebAssembly::CALL_INDIRECT_v4i32_S:
+  case WebAssembly::CALL_INDIRECT_v2i64:
+  case WebAssembly::CALL_INDIRECT_v2i64_S:
   case WebAssembly::CALL_INDIRECT_v4f32:
+  case WebAssembly::CALL_INDIRECT_v4f32_S:
+  case WebAssembly::CALL_INDIRECT_v2f64:
+  case WebAssembly::CALL_INDIRECT_v2f64_S:
   case WebAssembly::CALL_INDIRECT_EXCEPT_REF:
+  case WebAssembly::CALL_INDIRECT_EXCEPT_REF_S:
     return true;
   default:
     return false;
@@ -117,18 +169,54 @@ bool WebAssembly::isCallIndirect(const MachineInstr &MI) {
 unsigned WebAssembly::getCalleeOpNo(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::CALL_VOID:
+  case WebAssembly::CALL_VOID_S:
   case WebAssembly::CALL_INDIRECT_VOID:
+  case WebAssembly::CALL_INDIRECT_VOID_S:
     return 0;
   case WebAssembly::CALL_I32:
+  case WebAssembly::CALL_I32_S:
   case WebAssembly::CALL_I64:
+  case WebAssembly::CALL_I64_S:
   case WebAssembly::CALL_F32:
+  case WebAssembly::CALL_F32_S:
   case WebAssembly::CALL_F64:
+  case WebAssembly::CALL_F64_S:
+  case WebAssembly::CALL_v16i8:
+  case WebAssembly::CALL_v16i8_S:
+  case WebAssembly::CALL_v8i16:
+  case WebAssembly::CALL_v8i16_S:
+  case WebAssembly::CALL_v4i32:
+  case WebAssembly::CALL_v4i32_S:
+  case WebAssembly::CALL_v2i64:
+  case WebAssembly::CALL_v2i64_S:
+  case WebAssembly::CALL_v4f32:
+  case WebAssembly::CALL_v4f32_S:
+  case WebAssembly::CALL_v2f64:
+  case WebAssembly::CALL_v2f64_S:
   case WebAssembly::CALL_EXCEPT_REF:
+  case WebAssembly::CALL_EXCEPT_REF_S:
   case WebAssembly::CALL_INDIRECT_I32:
+  case WebAssembly::CALL_INDIRECT_I32_S:
   case WebAssembly::CALL_INDIRECT_I64:
+  case WebAssembly::CALL_INDIRECT_I64_S:
   case WebAssembly::CALL_INDIRECT_F32:
+  case WebAssembly::CALL_INDIRECT_F32_S:
   case WebAssembly::CALL_INDIRECT_F64:
+  case WebAssembly::CALL_INDIRECT_F64_S:
+  case WebAssembly::CALL_INDIRECT_v16i8:
+  case WebAssembly::CALL_INDIRECT_v16i8_S:
+  case WebAssembly::CALL_INDIRECT_v8i16:
+  case WebAssembly::CALL_INDIRECT_v8i16_S:
+  case WebAssembly::CALL_INDIRECT_v4i32:
+  case WebAssembly::CALL_INDIRECT_v4i32_S:
+  case WebAssembly::CALL_INDIRECT_v2i64:
+  case WebAssembly::CALL_INDIRECT_v2i64_S:
+  case WebAssembly::CALL_INDIRECT_v4f32:
+  case WebAssembly::CALL_INDIRECT_v4f32_S:
+  case WebAssembly::CALL_INDIRECT_v2f64:
+  case WebAssembly::CALL_INDIRECT_v2f64_S:
   case WebAssembly::CALL_INDIRECT_EXCEPT_REF:
+  case WebAssembly::CALL_INDIRECT_EXCEPT_REF_S:
     return 1;
   default:
     llvm_unreachable("Not a call instruction");
@@ -138,11 +226,17 @@ unsigned WebAssembly::getCalleeOpNo(const MachineInstr &MI) {
 bool WebAssembly::isMarker(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::BLOCK:
+  case WebAssembly::BLOCK_S:
   case WebAssembly::END_BLOCK:
+  case WebAssembly::END_BLOCK_S:
   case WebAssembly::LOOP:
+  case WebAssembly::LOOP_S:
   case WebAssembly::END_LOOP:
+  case WebAssembly::END_LOOP_S:
   case WebAssembly::TRY:
+  case WebAssembly::TRY_S:
   case WebAssembly::END_TRY:
+  case WebAssembly::END_TRY_S:
     return true;
   default:
     return false;
@@ -152,7 +246,9 @@ bool WebAssembly::isMarker(const MachineInstr &MI) {
 bool WebAssembly::isThrow(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::THROW_I32:
+  case WebAssembly::THROW_I32_S:
   case WebAssembly::THROW_I64:
+  case WebAssembly::THROW_I64_S:
     return true;
   default:
     return false;
@@ -162,7 +258,9 @@ bool WebAssembly::isThrow(const MachineInstr &MI) {
 bool WebAssembly::isRethrow(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::RETHROW:
+  case WebAssembly::RETHROW_S:
   case WebAssembly::RETHROW_TO_CALLER:
+  case WebAssembly::RETHROW_TO_CALLER_S:
     return true;
   default:
     return false;
@@ -172,8 +270,11 @@ bool WebAssembly::isRethrow(const MachineInstr &MI) {
 bool WebAssembly::isCatch(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::CATCH_I32:
+  case WebAssembly::CATCH_I32_S:
   case WebAssembly::CATCH_I64:
+  case WebAssembly::CATCH_I64_S:
   case WebAssembly::CATCH_ALL:
+  case WebAssembly::CATCH_ALL_S:
     return true;
   default:
     return false;
@@ -183,8 +284,11 @@ bool WebAssembly::isCatch(const MachineInstr &MI) {
 bool WebAssembly::mayThrow(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case WebAssembly::THROW_I32:
+  case WebAssembly::THROW_I32_S:
   case WebAssembly::THROW_I64:
+  case WebAssembly::THROW_I64_S:
   case WebAssembly::RETHROW:
+  case WebAssembly::RETHROW_S:
     return true;
   }
   if (isCallIndirect(MI))
@@ -212,7 +316,9 @@ bool WebAssembly::isCatchTerminatePad(const MachineBasicBlock &MBB) {
   bool SeenCatch = false;
   for (auto &MI : MBB) {
     if (MI.getOpcode() == WebAssembly::CATCH_I32 ||
-        MI.getOpcode() == WebAssembly::CATCH_I64)
+        MI.getOpcode() == WebAssembly::CATCH_I64 ||
+        MI.getOpcode() == WebAssembly::CATCH_I32_S ||
+        MI.getOpcode() == WebAssembly::CATCH_I64_S)
       SeenCatch = true;
     if (SeenCatch && MI.isCall()) {
       const MachineOperand &CalleeOp = MI.getOperand(getCalleeOpNo(MI));
@@ -229,7 +335,8 @@ bool WebAssembly::isCatchAllTerminatePad(const MachineBasicBlock &MBB) {
     return false;
   bool SeenCatchAll = false;
   for (auto &MI : MBB) {
-    if (MI.getOpcode() == WebAssembly::CATCH_ALL)
+    if (MI.getOpcode() == WebAssembly::CATCH_ALL ||
+        MI.getOpcode() == WebAssembly::CATCH_ALL_S)
       SeenCatchAll = true;
     if (SeenCatchAll && MI.isCall()) {
       const MachineOperand &CalleeOp = MI.getOperand(getCalleeOpNo(MI));