vendor/llvm/llvm-trunk-r302418

49 files changed, 5108 insertions, 642 deletions

diff --git a/test/CodeGen/X86/2014-08-29-CompactUnwind.ll b/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
index e7e8bb724fc0..f6d6bd3ed6f7 100644
--- a/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
+++ b/test/CodeGen/X86/2014-08-29-CompactUnwind.ll
@@ -24,7 +24,7 @@ target triple = "x86_64-apple-macosx10.9.0"
 ; CHECK-NOT: {{compact encoding:.*0x0309f800}}
 ; CHECK: {{compact encoding:.*0x030df800}}
 
-define void @__asan_report_error() #0 {
+define void @__asan_report_error(i64 %step) #0 {
   %str.i = alloca i64, align 8
   %stack = alloca [256 x i64], align 8
   br label %print_shadow_bytes.exit.i
@@ -37,7 +37,7 @@ print_shadow_bytes.exit.i: ; preds = %print_shadow_bytes.exit.i, %0
   %reg17 = shl i64 %iv.i, 1
   %reg19 = inttoptr i64 %reg17 to i8*
   call void (i64*, i8*, ...) @append(i64* %str.i, i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str2, i64 0, i64 0), i8* %reg16, i8* %reg19)
-  %iv.next.i = add nsw i64 %iv.i, 0
+  %iv.next.i = add nsw i64 %iv.i, %step
   br label %print_shadow_bytes.exit.i
 }
 
diff --git a/test/CodeGen/X86/GlobalISel/gep.ll b/test/CodeGen/X86/GlobalISel/gep.ll
new file mode 100644
index 000000000000..bc5b0152b24a
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/gep.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -global-isel < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64_GISEL
+; RUN: llc -mtriple=x86_64-linux-gnu              < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+
+define i32* @test_gep_i8(i32 *%arr, i8 %ind) {
+; X64_GISEL-LABEL: test_gep_i8:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    movsbq %sil, %rcx
+; X64_GISEL-NEXT:    imulq %rax, %rcx
+; X64_GISEL-NEXT:    leaq (%rdi,%rcx), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i8:
+; X64:       # BB#0:
+; X64-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; X64-NEXT:    movsbq %sil, %rax
+; X64-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i8 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i8_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i8_const:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $80, %rax
+; X64_GISEL-NEXT:    leaq (%rdi,%rax), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i8_const:
+; X64:       # BB#0:
+; X64-NEXT:    leaq 80(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i8 20
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i16(i32 *%arr, i16 %ind) {
+; X64_GISEL-LABEL: test_gep_i16:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    movswq %si, %rcx
+; X64_GISEL-NEXT:    imulq %rax, %rcx
+; X64_GISEL-NEXT:    leaq (%rdi,%rcx), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i16:
+; X64:       # BB#0:
+; X64-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; X64-NEXT:    movswq %si, %rax
+; X64-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i16 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i16_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i16_const:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $80, %rax
+; X64_GISEL-NEXT:    leaq (%rdi,%rax), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i16_const:
+; X64:       # BB#0:
+; X64-NEXT:    leaq 80(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i16 20
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i32(i32 *%arr, i32 %ind) {
+; X64_GISEL-LABEL: test_gep_i32:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    movslq %esi, %rcx
+; X64_GISEL-NEXT:    imulq %rax, %rcx
+; X64_GISEL-NEXT:    leaq (%rdi,%rcx), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i32:
+; X64:       # BB#0:
+; X64-NEXT:    movslq %esi, %rax
+; X64-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i32 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i32_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i32_const:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $20, %rax
+; X64_GISEL-NEXT:    leaq (%rdi,%rax), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i32_const:
+; X64:       # BB#0:
+; X64-NEXT:    leaq 20(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i32 5
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i64(i32 *%arr, i64 %ind) {
+; X64_GISEL-LABEL: test_gep_i64:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $4, %rax
+; X64_GISEL-NEXT:    imulq %rsi, %rax
+; X64_GISEL-NEXT:    leaq (%rdi,%rax), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i64:
+; X64:       # BB#0:
+; X64-NEXT:    leaq (%rdi,%rsi,4), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i64 %ind
+  ret i32* %arrayidx
+}
+
+define i32* @test_gep_i64_const(i32 *%arr) {
+; X64_GISEL-LABEL: test_gep_i64_const:
+; X64_GISEL:       # BB#0:
+; X64_GISEL-NEXT:    movq $20, %rax
+; X64_GISEL-NEXT:    leaq (%rdi,%rax), %rax
+; X64_GISEL-NEXT:    retq
+;
+; X64-LABEL: test_gep_i64_const:
+; X64:       # BB#0:
+; X64-NEXT:    leaq 20(%rdi), %rax
+; X64-NEXT:    retq
+  %arrayidx = getelementptr i32, i32* %arr, i64 5
+  ret i32* %arrayidx
+}
+
diff --git a/test/CodeGen/X86/GlobalISel/legalize-gep.mir b/test/CodeGen/X86/GlobalISel/legalize-gep.mir
new file mode 100644
index 000000000000..4fdb9b910ad7
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-gep.mir
@@ -0,0 +1,101 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer %s -o - | FileCheck %s
+
+--- |
+  define void @test_gep_i8() {
+    %arrayidx = getelementptr i32, i32* undef, i8 5
+    ret void
+  }
+
+  define void @test_gep_i16() {
+    %arrayidx = getelementptr i32, i32* undef, i16 5
+    ret void
+  }
+
+  define void @test_gep_i32() {
+    %arrayidx = getelementptr i32, i32* undef, i32 5
+    ret void
+  }
+
+  define void @test_gep_i64() {
+    %arrayidx = getelementptr i32, i32* undef, i64 5
+    ret void
+  }
+...
+---
+name:            test_gep_i8
+# CHECK-LABEL: name:  test_gep_i8
+legalized:       false
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:          %0(p0) = IMPLICIT_DEF
+# CHECK-NEXT:     %1(s8) = G_CONSTANT i8 20
+# CHECK-NEXT:     %3(s32) = G_SEXT %1(s8)
+# CHECK-NEXT:     %2(p0) = G_GEP %0, %3(s32)
+# CHECK-NEXT:     RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    %0(p0) = IMPLICIT_DEF
+    %1(s8) = G_CONSTANT i8 20
+    %2(p0) = G_GEP %0, %1(s8)
+    RET 0
+...
+---
+name:            test_gep_i16
+# CHECK-LABEL: name:  test_gep_i16
+legalized:       false
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:          %0(p0) = IMPLICIT_DEF
+# CHECK-NEXT:     %1(s16) = G_CONSTANT i16 20
+# CHECK-NEXT:     %3(s32) = G_SEXT %1(s16)
+# CHECK-NEXT:     %2(p0) = G_GEP %0, %3(s32)
+# CHECK-NEXT:     RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    %0(p0) = IMPLICIT_DEF
+    %1(s16) = G_CONSTANT i16 20
+    %2(p0) = G_GEP %0, %1(s16)
+    RET 0
+...
+---
+name:            test_gep_i32
+# CHECK-LABEL: name:  test_gep_i32
+legalized:       false
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:          %0(p0) = IMPLICIT_DEF
+# CHECK-NEXT:     %1(s32) = G_CONSTANT i32 20
+# CHECK-NEXT:     %2(p0) = G_GEP %0, %1(s32)
+# CHECK-NEXT:     RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    %0(p0) = IMPLICIT_DEF
+    %1(s32) = G_CONSTANT i32 20
+    %2(p0) = G_GEP %0, %1(s32)
+    RET 0
+...
+---
+name:            test_gep_i64
+# CHECK-LABEL: name:  test_gep_i64
+legalized:       false
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:          %0(p0) = IMPLICIT_DEF
+# CHECK-NEXT:     %1(s64) = G_CONSTANT i64 20
+# CHECK-NEXT:     %2(p0) = G_GEP %0, %1(s64)
+# CHECK-NEXT:     RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    %0(p0) = IMPLICIT_DEF
+    %1(s64) = G_CONSTANT i64 20
+    %2(p0) = G_GEP %0, %1(s64)
+    RET 0
+...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir b/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
new file mode 100644
index 000000000000..0d66a6384107
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
@@ -0,0 +1,115 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer %s -o - | FileCheck %s
+
+--- |
+  define i16 @test_mul_i16(i16 %arg1, i16 %arg2) {
+    %ret = mul i16 %arg1, %arg2
+    ret i16 %ret
+  }
+
+  define i32 @test_mul_i32(i32 %arg1, i32 %arg2) {
+    %ret = mul i32 %arg1, %arg2
+    ret i32 %ret
+  }
+
+  define i64 @test_mul_i64(i64 %arg1, i64 %arg2) {
+    %ret = mul i64 %arg1, %arg2
+    ret i64 %ret
+  }
+
+...
+---
+name:            test_mul_i16
+# CHECK-LABEL: name:  test_mul_i16
+alignment:       4
+legalized:       false
+regBankSelected: false
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: _ }
+# CHECK-NEXT:   - { id: 1, class: _ }
+# CHECK-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:      body:             |
+# CHECK-NEXT:   bb.0 (%ir-block.0):
+# CHECK-NEXT:     %0(s16) = COPY %edi
+# CHECK-NEXT:     %1(s16) = COPY %esi
+# CHECK-NEXT:     %2(s16) = G_MUL %0, %1
+# CHECK-NEXT:     %ax = COPY %2(s16)
+# CHECK-NEXT:     RET 0, implicit %ax
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %edi, %esi
+
+    %0(s16) = COPY %edi
+    %1(s16) = COPY %esi
+    %2(s16) = G_MUL %0, %1
+    %ax = COPY %2(s16)
+    RET 0, implicit %ax
+
+...
+---
+name:            test_mul_i32
+# CHECK-LABEL: name:  test_mul_i32
+alignment:       4
+legalized:       false
+regBankSelected: false
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: _ }
+# CHECK-NEXT:   - { id: 1, class: _ }
+# CHECK-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:      body:             |
+# CHECK-NEXT:   bb.0 (%ir-block.0):
+# CHECK-NEXT:     %0(s32) = COPY %edi
+# CHECK-NEXT:     %1(s32) = COPY %esi
+# CHECK-NEXT:     %2(s32) = G_MUL %0, %1
+# CHECK-NEXT:     %eax = COPY %2(s32)
+# CHECK-NEXT:     RET 0, implicit %eax
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %edi, %esi
+
+    %0(s32) = COPY %edi
+    %1(s32) = COPY %esi
+    %2(s32) = G_MUL %0, %1
+    %eax = COPY %2(s32)
+    RET 0, implicit %eax
+
+...
+---
+name:            test_mul_i64
+# CHECK-LABEL: name:  test_mul_i64
+alignment:       4
+legalized:       false
+regBankSelected: false
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: _ }
+# CHECK-NEXT:   - { id: 1, class: _ }
+# CHECK-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# CHECK:      body:             |
+# CHECK-NEXT:   bb.0 (%ir-block.0):
+# CHECK-NEXT:     %0(s64) = COPY %rdi
+# CHECK-NEXT:     %1(s64) = COPY %rsi
+# CHECK-NEXT:     %2(s64) = G_MUL %0, %1
+# CHECK-NEXT:     %rax = COPY %2(s64)
+# CHECK-NEXT:     RET 0, implicit %rax
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %rsi
+
+    %0(s64) = COPY %rdi
+    %1(s64) = COPY %rsi
+    %2(s64) = G_MUL %0, %1
+    %rax = COPY %2(s64)
+    RET 0, implicit %rax
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir b/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir
new file mode 100644
index 000000000000..be62832b008a
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir
@@ -0,0 +1,111 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL
+--- |
+  define <8 x i16> @test_mul_v8i16(<8 x i16> %arg1, <8 x i16> %arg2) #0 {
+    %ret = mul <8 x i16> %arg1, %arg2
+    ret <8 x i16> %ret
+  }
+
+  define <4 x i32> @test_mul_v4i32(<4 x i32> %arg1, <4 x i32> %arg2) #0 {
+    %ret = mul <4 x i32> %arg1, %arg2
+    ret <4 x i32> %ret
+  }
+
+  define <2 x i64> @test_mul_v2i64(<2 x i64> %arg1, <2 x i64> %arg2) #1 {
+    %ret = mul <2 x i64> %arg1, %arg2
+    ret <2 x i64> %ret
+  }
+
+  attributes #0 = { "target-features"="+sse4.1" }
+  attributes #1 = { "target-features"="+sse4.1,+avx512vl,+avx512f,+avx512dq" }
+
+...
+---
+name:            test_mul_v8i16
+# ALL-LABEL: name:  test_mul_v8i16
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<8 x s16>) = COPY %xmm0
+# ALL-NEXT:     %1(<8 x s16>) = COPY %xmm1
+# ALL-NEXT:     %2(<8 x s16>) = G_MUL %0, %1
+# ALL-NEXT:     %xmm0 = COPY %2(<8 x s16>)
+# ALL-NEXT:     RET 0, implicit %xmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<8 x s16>) = COPY %xmm0
+    %1(<8 x s16>) = COPY %xmm1
+    %2(<8 x s16>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<8 x s16>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v4i32
+# ALL-LABEL: name:  test_mul_v4i32
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<4 x s32>) = COPY %xmm0
+# ALL-NEXT:     %1(<4 x s32>) = COPY %xmm1
+# ALL-NEXT:     %2(<4 x s32>) = G_MUL %0, %1
+# ALL-NEXT:     %xmm0 = COPY %2(<4 x s32>)
+# ALL-NEXT:     RET 0, implicit %xmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<4 x s32>) = COPY %xmm0
+    %1(<4 x s32>) = COPY %xmm1
+    %2(<4 x s32>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<4 x s32>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v2i64
+# ALL-LABEL: name:  test_mul_v2i64
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<2 x s64>) = COPY %xmm0
+# ALL-NEXT:     %1(<2 x s64>) = COPY %xmm1
+# ALL-NEXT:     %2(<2 x s64>) = G_MUL %0, %1
+# ALL-NEXT:     %xmm0 = COPY %2(<2 x s64>)
+# ALL-NEXT:     RET 0, implicit %xmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<2 x s64>) = COPY %xmm0
+    %1(<2 x s64>) = COPY %xmm1
+    %2(<2 x s64>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<2 x s64>)
+    RET 0, implicit %xmm0
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir b/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir
new file mode 100644
index 000000000000..d99303c3ba3b
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir
@@ -0,0 +1,111 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL
+--- |
+  define <16 x i16> @test_mul_v16i16(<16 x i16> %arg1, <16 x i16> %arg2) #0 {
+    %ret = mul <16 x i16> %arg1, %arg2
+    ret <16 x i16> %ret
+  }
+
+  define <8 x i32> @test_mul_v8i32(<8 x i32> %arg1, <8 x i32> %arg2) #0 {
+    %ret = mul <8 x i32> %arg1, %arg2
+    ret <8 x i32> %ret
+  }
+
+  define <4 x i64> @test_mul_v4i64(<4 x i64> %arg1, <4 x i64> %arg2) #1 {
+    %ret = mul <4 x i64> %arg1, %arg2
+    ret <4 x i64> %ret
+  }
+
+  attributes #0 = { "target-features"="+avx2" }
+  attributes #1 = { "target-features"="+avx2,+avx512vl,+avx512f,+avx512dq" }
+
+...
+---
+name:            test_mul_v16i16
+# ALL-LABEL: name:  test_mul_v16i16
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<16 x s16>) = COPY %ymm0
+# ALL-NEXT:     %1(<16 x s16>) = COPY %ymm1
+# ALL-NEXT:     %2(<16 x s16>) = G_MUL %0, %1
+# ALL-NEXT:     %ymm0 = COPY %2(<16 x s16>)
+# ALL-NEXT:     RET 0, implicit %ymm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<16 x s16>) = COPY %ymm0
+    %1(<16 x s16>) = COPY %ymm1
+    %2(<16 x s16>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<16 x s16>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v8i32
+# ALL-LABEL: name:  test_mul_v8i32
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<8 x s32>) = COPY %ymm0
+# ALL-NEXT:     %1(<8 x s32>) = COPY %ymm1
+# ALL-NEXT:     %2(<8 x s32>) = G_MUL %0, %1
+# ALL-NEXT:     %ymm0 = COPY %2(<8 x s32>)
+# ALL-NEXT:     RET 0, implicit %ymm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<8 x s32>) = COPY %ymm0
+    %1(<8 x s32>) = COPY %ymm1
+    %2(<8 x s32>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<8 x s32>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v4i64
+# ALL-LABEL: name:  test_mul_v4i64
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<4 x s64>) = COPY %ymm0
+# ALL-NEXT:     %1(<4 x s64>) = COPY %ymm1
+# ALL-NEXT:     %2(<4 x s64>) = G_MUL %0, %1
+# ALL-NEXT:     %ymm0 = COPY %2(<4 x s64>)
+# ALL-NEXT:     RET 0, implicit %ymm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<4 x s64>) = COPY %ymm0
+    %1(<4 x s64>) = COPY %ymm1
+    %2(<4 x s64>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<4 x s64>)
+    RET 0, implicit %ymm0
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir b/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir
new file mode 100644
index 000000000000..24eefd30c2ac
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir
@@ -0,0 +1,113 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL
+
+--- |
+  define <32 x i16> @test_mul_v32i16(<32 x i16> %arg1, <32 x i16> %arg2) #0 {
+    %ret = mul <32 x i16> %arg1, %arg2
+    ret <32 x i16> %ret
+  }
+
+  define <16 x i32> @test_mul_v16i32(<16 x i32> %arg1, <16 x i32> %arg2) #1 {
+    %ret = mul <16 x i32> %arg1, %arg2
+    ret <16 x i32> %ret
+  }
+
+  define <8 x i64> @test_mul_v8i64(<8 x i64> %arg1, <8 x i64> %arg2) #2 {
+    %ret = mul <8 x i64> %arg1, %arg2
+    ret <8 x i64> %ret
+  }
+
+  attributes #0 = { "target-features"="+avx512f,+avx512bw" }
+  attributes #1 = { "target-features"="+avx512f" }
+  attributes #2 = { "target-features"="+avx512f,+avx512dq" }
+
+...
+---
+name:            test_mul_v32i16
+# ALL-LABEL: name:  test_mul_v32i16
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<32 x s16>) = COPY %zmm0
+# ALL-NEXT:     %1(<32 x s16>) = COPY %zmm1
+# ALL-NEXT:     %2(<32 x s16>) = G_MUL %0, %1
+# ALL-NEXT:     %zmm0 = COPY %2(<32 x s16>)
+# ALL-NEXT:     RET 0, implicit %zmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %zmm0, %zmm1
+
+    %0(<32 x s16>) = COPY %zmm0
+    %1(<32 x s16>) = COPY %zmm1
+    %2(<32 x s16>) = G_MUL %0, %1
+    %zmm0 = COPY %2(<32 x s16>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_mul_v16i32
+# ALL-LABEL: name:  test_mul_v16i32
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<16 x s32>) = COPY %zmm0
+# ALL-NEXT:     %1(<16 x s32>) = COPY %zmm1
+# ALL-NEXT:     %2(<16 x s32>) = G_MUL %0, %1
+# ALL-NEXT:     %zmm0 = COPY %2(<16 x s32>)
+# ALL-NEXT:     RET 0, implicit %zmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %zmm0, %zmm1
+
+    %0(<16 x s32>) = COPY %zmm0
+    %1(<16 x s32>) = COPY %zmm1
+    %2(<16 x s32>) = G_MUL %0, %1
+    %zmm0 = COPY %2(<16 x s32>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_mul_v8i64
+# ALL-LABEL: name:  test_mul_v8i64
+alignment:       4
+legalized:       false
+regBankSelected: false
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: _ }
+# ALL-NEXT:   - { id: 1, class: _ }
+# ALL-NEXT:   - { id: 2, class: _ }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+# ALL:          %0(<8 x s64>) = COPY %zmm0
+# ALL-NEXT:     %1(<8 x s64>) = COPY %zmm1
+# ALL-NEXT:     %2(<8 x s64>) = G_MUL %0, %1
+# ALL-NEXT:     %zmm0 = COPY %2(<8 x s64>)
+# ALL-NEXT:     RET 0, implicit %zmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %zmm0, %zmm1
+
+    %0(<8 x s64>) = COPY %zmm0
+    %1(<8 x s64>) = COPY %zmm1
+    %2(<8 x s64>) = G_MUL %0, %1
+    %zmm0 = COPY %2(<8 x s64>)
+    RET 0, implicit %zmm0
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/mul-scalar.ll b/test/CodeGen/X86/GlobalISel/mul-scalar.ll
new file mode 100644
index 000000000000..529e81c43304
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/mul-scalar.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -global-isel < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+
+;TODO: instruction selection not supported yet
+;define i8 @test_mul_i8(i8 %arg1, i8 %arg2) {
+;  %ret = mul i8 %arg1, %arg2
+;  ret i8 %ret
+;}
+
+define i16 @test_mul_i16(i16 %arg1, i16 %arg2) {
+; X64-LABEL: test_mul_i16:
+; X64:       # BB#0:
+; X64-NEXT:    imulw %di, %si
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    retq
+  %ret = mul i16 %arg1, %arg2
+  ret i16 %ret
+}
+
+define i32 @test_mul_i32(i32 %arg1, i32 %arg2) {
+; X64-LABEL: test_mul_i32:
+; X64:       # BB#0:
+; X64-NEXT:    imull %edi, %esi
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    retq
+  %ret = mul i32 %arg1, %arg2
+  ret i32 %ret
+}
+
+define i64 @test_mul_i64(i64 %arg1, i64 %arg2) {
+; X64-LABEL: test_mul_i64:
+; X64:       # BB#0:
+; X64-NEXT:    imulq %rdi, %rsi
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    retq
+  %ret = mul i64 %arg1, %arg2
+  ret i64 %ret
+}
+
diff --git a/test/CodeGen/X86/GlobalISel/mul-vec.ll b/test/CodeGen/X86/GlobalISel/mul-vec.ll
new file mode 100644
index 000000000000..83615a718528
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/mul-vec.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -mcpu=skx -global-isel < %s -o - | FileCheck %s --check-prefix=SKX
+
+define <8 x i16> @test_mul_v8i16(<8 x i16> %arg1, <8 x i16> %arg2) {
+; SKX-LABEL: test_mul_v8i16:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %ret = mul <8 x i16> %arg1, %arg2
+  ret <8 x i16> %ret
+}
+
+define <4 x i32> @test_mul_v4i32(<4 x i32> %arg1, <4 x i32> %arg2) {
+; SKX-LABEL: test_mul_v4i32:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %ret = mul <4 x i32> %arg1, %arg2
+  ret <4 x i32> %ret
+}
+
+define <2 x i64> @test_mul_v2i64(<2 x i64> %arg1, <2 x i64> %arg2) {
+; SKX-LABEL: test_mul_v2i64:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
+; SKX-NEXT:    retq
+  %ret = mul <2 x i64> %arg1, %arg2
+  ret <2 x i64> %ret
+}
+
+define <16 x i16> @test_mul_v16i16(<16 x i16> %arg1, <16 x i16> %arg2) {
+; SKX-LABEL: test_mul_v16i16:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    retq
+  %ret = mul <16 x i16> %arg1, %arg2
+  ret <16 x i16> %ret
+}
+
+define <8 x i32> @test_mul_v8i32(<8 x i32> %arg1, <8 x i32> %arg2) {
+; SKX-LABEL: test_mul_v8i32:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    retq
+  %ret = mul <8 x i32> %arg1, %arg2
+  ret <8 x i32> %ret
+}
+
+define <4 x i64> @test_mul_v4i64(<4 x i64> %arg1, <4 x i64> %arg2) {
+; SKX-LABEL: test_mul_v4i64:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmullq %ymm1, %ymm0, %ymm0
+; SKX-NEXT:    retq
+  %ret = mul <4 x i64> %arg1, %arg2
+  ret <4 x i64> %ret
+}
+
+define <32 x i16> @test_mul_v32i16(<32 x i16> %arg1, <32 x i16> %arg2) {
+; SKX-LABEL: test_mul_v32i16:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %ret = mul <32 x i16> %arg1, %arg2
+  ret <32 x i16> %ret
+}
+
+define <16 x i32> @test_mul_v16i32(<16 x i32> %arg1, <16 x i32> %arg2) {
+; SKX-LABEL: test_mul_v16i32:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %ret = mul <16 x i32> %arg1, %arg2
+  ret <16 x i32> %ret
+}
+
+define <8 x i64> @test_mul_v8i64(<8 x i64> %arg1, <8 x i64> %arg2) {
+; SKX-LABEL: test_mul_v8i64:
+; SKX:       # BB#0:
+; SKX-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %ret = mul <8 x i64> %arg1, %arg2
+  ret <8 x i64> %ret
+}
+
diff --git a/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir b/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
new file mode 100644
index 000000000000..446db56b992c
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
@@ -0,0 +1,31 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx2 --global-isel                       -run-pass=regbankselect %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=FAST
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx2 --global-isel -regbankselect-greedy -run-pass=regbankselect %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=GREEDY
+
+--- |
+  define void @test_mul_vec256() {
+    ret void
+  }
+...
+---
+name:            test_mul_vec256
+alignment:       4
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+# CHECK-LABEL: name:            test_mul_vec256
+# CHECK: registers:
+# CHECK:  - { id: 0, class: vecr }
+# CHECK:  - { id: 1, class: vecr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+
+    %0(<8 x s32>) = IMPLICIT_DEF
+    %1(<8 x s32>) = G_MUL %0, %0
+    RET 0
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir b/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
new file mode 100644
index 000000000000..f824ee12dcfb
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
@@ -0,0 +1,33 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f -global-isel                       -run-pass=regbankselect %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=FAST
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f -global-isel -regbankselect-greedy -run-pass=regbankselect %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=GREEDY
+
+--- |
+
+  define void @test_mul_vec512() {
+    ret void
+  }
+
+...
+---
+name:            test_mul_vec512
+alignment:       4
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+# CHECK-LABEL: name:            test_mul_vec512
+# CHECK: registers:
+# CHECK:  - { id: 0, class: vecr }
+# CHECK:  - { id: 1, class: vecr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+
+    %0(<16 x s32>) = IMPLICIT_DEF
+    %1(<16 x s32>) = G_MUL %0, %0
+    RET 0
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir b/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir
index 8e04239041a8..3a65a9003773 100644
--- a/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir
+++ b/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir
@@ -27,6 +27,10 @@
     ret i64 %ret
   }
 
+  define void @test_mul_gpr() {
+    ret void
+  }
+
   define float @test_add_float(float %arg1, float %arg2) {
     %ret = fadd float %arg1, %arg2
     ret float %ret
@@ -110,6 +114,12 @@
     ret void
   }
 
+  define void @test_gep() {
+    %p1 = getelementptr i32, i32* undef, i32 5
+    %p2 = getelementptr i32, i32* undef, i64 5
+    ret void
+  }
+
 ...
 ---
 name:            test_add_i8
@@ -220,6 +230,45 @@ body:             |
 
 ...
 ---
+name:            test_mul_gpr
+alignment:       4
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+# CHECK-LABEL: name:            test_mul_gpr
+# CHECK: registers:
+# CHECK:  - { id: 0, class: gpr }
+# CHECK:  - { id: 1, class: gpr }
+# CHECK:  - { id: 2, class: gpr }
+# CHECK:  - { id: 3, class: gpr }
+# CHECK:  - { id: 4, class: gpr }
+# CHECK:  - { id: 5, class: gpr }
+# CHECK:  - { id: 6, class: gpr }
+# CHECK:  - { id: 7, class: gpr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+  - { id: 4, class: _ }
+  - { id: 5, class: _ }
+  - { id: 6, class: _ }
+  - { id: 7, class: _ }  
+body:             |
+  bb.1 (%ir-block.0):
+    
+    %0(s64) = IMPLICIT_DEF
+    %1(s32) = IMPLICIT_DEF
+    %2(s16) = IMPLICIT_DEF
+    %3(s8)  = IMPLICIT_DEF
+    %4(s64) = G_MUL %0, %0
+    %5(s32) = G_MUL %1, %1
+    %6(s16) = G_MUL %2, %2
+    %7(s8)  = G_MUL %3, %3
+    RET 0
+...
+---
 name:            test_add_float
 alignment:       4
 legalized:       true
@@ -660,3 +709,29 @@ body:             |
     RET 0
 
 ...
+---
+name:            test_gep
+legalized:       true
+# CHECK-LABEL: name:            test_gep
+# CHECK: registers:
+# CHECK-NEXT:  - { id: 0, class: gpr }
+# CHECK-NEXT:  - { id: 1, class: gpr }
+# CHECK-NEXT:  - { id: 2, class: gpr }
+# CHECK-NEXT:  - { id: 3, class: gpr }
+# CHECK-NEXT:  - { id: 4, class: gpr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+  - { id: 4, class: _ }
+body:             |
+  bb.0 (%ir-block.0):
+    %0(p0) = IMPLICIT_DEF
+    %1(s32) = G_CONSTANT i32 20
+    %2(p0) = G_GEP %0, %1(s32)
+    %3(s64) = G_CONSTANT i64 20
+    %4(p0) = G_GEP %0, %3(s64)
+    RET 0
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/select-gep.mir b/test/CodeGen/X86/GlobalISel/select-gep.mir
new file mode 100644
index 000000000000..2c89b7057c3d
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/select-gep.mir
@@ -0,0 +1,37 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=instruction-select %s -o - | FileCheck %s --check-prefix=CHECK
+
+--- |
+  define i32* @test_gep_i32(i32* %arr) {
+    %arrayidx = getelementptr i32, i32* %arr, i32 5
+    ret i32* %arrayidx
+  }
+...
+---
+name:            test_gep_i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK-LABEL: name:            test_gep_i32
+# CHECK: registers:
+# CHECK-NEXT: - { id: 0, class: gr64 }
+# CHECK-NEXT: - { id: 1, class: gr64_nosp }
+# CHECK-NEXT: - { id: 2, class: gr64 }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+# CHECK:  body:
+# CHECK:       %1 = MOV64ri32 20
+# CHECK-NEXT:  %2 = LEA64r %0, 1, %1, 0, _
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(s64) = G_CONSTANT i64 20
+    %2(p0) = G_GEP %0, %1(s64)
+    %rax = COPY %2(p0)
+    RET 0, implicit %rax
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/select-mul-scalar.mir b/test/CodeGen/X86/GlobalISel/select-mul-scalar.mir
new file mode 100644
index 000000000000..34a77acc2d1e
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/select-mul-scalar.mir
@@ -0,0 +1,112 @@
+# RUN: llc -mtriple=x86_64-linux-gnu                                  -global-isel -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=ALL 
+
+--- |
+  define i16 @test_mul_i16(i16 %arg1, i16 %arg2) {
+    %ret = mul i16 %arg1, %arg2
+    ret i16 %ret
+  }
+
+  define i32 @test_mul_i32(i32 %arg1, i32 %arg2) {
+    %ret = mul i32 %arg1, %arg2
+    ret i32 %ret
+  }
+
+  define i64 @test_mul_i64(i64 %arg1, i64 %arg2) {
+    %ret = mul i64 %arg1, %arg2
+    ret i64 %ret
+  }
+
+...
+---
+name:            test_mul_i16
+# ALL-LABEL: name:  test_mul_i16
+alignment:       4
+legalized:       true
+regBankSelected: true
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: gr16 }
+# ALL-NEXT:   - { id: 1, class: gr16 }
+# ALL-NEXT:   - { id: 2, class: gr16 }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+# ALL:      body:             |
+# ALL:          %0 = COPY %di
+# ALL-NEXT:     %1 = COPY %si
+# ALL-NEXT:     %2 = IMUL16rr %0, %1, implicit-def %eflags
+# ALL-NEXT:     %ax = COPY %2
+# ALL-NEXT:     RET 0, implicit %ax
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %edi, %esi
+
+    %0(s16) = COPY %edi
+    %1(s16) = COPY %esi
+    %2(s16) = G_MUL %0, %1
+    %ax = COPY %2(s16)
+    RET 0, implicit %ax
+
+...
+---
+name:            test_mul_i32
+# ALL-LABEL: name:  test_mul_i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: gr32 }
+# ALL-NEXT:   - { id: 1, class: gr32 }
+# ALL-NEXT:   - { id: 2, class: gr32 }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+# ALL:      body:             |
+# ALL:          %0 = COPY %edi
+# ALL-NEXT:     %1 = COPY %esi
+# ALL-NEXT:     %2 = IMUL32rr %0, %1, implicit-def %eflags
+# ALL-NEXT:     %eax = COPY %2
+# ALL-NEXT:     RET 0, implicit %eax
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %edi, %esi
+
+    %0(s32) = COPY %edi
+    %1(s32) = COPY %esi
+    %2(s32) = G_MUL %0, %1
+    %eax = COPY %2(s32)
+    RET 0, implicit %eax
+
+...
+---
+name:            test_mul_i64
+# ALL-LABEL: name:  test_mul_i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+# ALL:      registers:
+# ALL-NEXT:   - { id: 0, class: gr64 }
+# ALL-NEXT:   - { id: 1, class: gr64 }
+# ALL-NEXT:   - { id: 2, class: gr64 }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+# ALL:      body:             |
+# ALL:          %0 = COPY %rdi
+# ALL-NEXT:     %1 = COPY %rsi
+# ALL-NEXT:     %2 = IMUL64rr %0, %1, implicit-def %eflags
+# ALL-NEXT:     %rax = COPY %2
+# ALL-NEXT:     RET 0, implicit %rax
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %rsi
+
+    %0(s64) = COPY %rdi
+    %1(s64) = COPY %rsi
+    %2(s64) = G_MUL %0, %1
+    %rax = COPY %2(s64)
+    RET 0, implicit %rax
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/select-mul-vec.mir b/test/CodeGen/X86/GlobalISel/select-mul-vec.mir
new file mode 100644
index 000000000000..5f8ab1e4f189
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/select-mul-vec.mir
@@ -0,0 +1,480 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  define <8 x i16> @test_mul_v8i16(<8 x i16> %arg1, <8 x i16> %arg2) #0 {
+    %ret = mul <8 x i16> %arg1, %arg2
+    ret <8 x i16> %ret
+  }
+
+  define <8 x i16> @test_mul_v8i16_avx(<8 x i16> %arg1, <8 x i16> %arg2) #1 {
+    %ret = mul <8 x i16> %arg1, %arg2
+    ret <8 x i16> %ret
+  }
+
+  define <8 x i16> @test_mul_v8i16_avx512bwvl(<8 x i16> %arg1, <8 x i16> %arg2) #2 {
+    %ret = mul <8 x i16> %arg1, %arg2
+    ret <8 x i16> %ret
+  }
+
+  define <4 x i32> @test_mul_v4i32(<4 x i32> %arg1, <4 x i32> %arg2) #3 {
+    %ret = mul <4 x i32> %arg1, %arg2
+    ret <4 x i32> %ret
+  }
+
+  define <4 x i32> @test_mul_v4i32_avx(<4 x i32> %arg1, <4 x i32> %arg2) #1 {
+    %ret = mul <4 x i32> %arg1, %arg2
+    ret <4 x i32> %ret
+  }
+
+  define <4 x i32> @test_mul_v4i32_avx512vl(<4 x i32> %arg1, <4 x i32> %arg2) #4 {
+    %ret = mul <4 x i32> %arg1, %arg2
+    ret <4 x i32> %ret
+  }
+
+  define <2 x i64> @test_mul_v2i64(<2 x i64> %arg1, <2 x i64> %arg2) #5 {
+    %ret = mul <2 x i64> %arg1, %arg2
+    ret <2 x i64> %ret
+  }
+
+  define <16 x i16> @test_mul_v16i16(<16 x i16> %arg1, <16 x i16> %arg2) #6 {
+    %ret = mul <16 x i16> %arg1, %arg2
+    ret <16 x i16> %ret
+  }
+
+  define <16 x i16> @test_mul_v16i16_avx512bwvl(<16 x i16> %arg1, <16 x i16> %arg2) #2 {
+    %ret = mul <16 x i16> %arg1, %arg2
+    ret <16 x i16> %ret
+  }
+
+  define <8 x i32> @test_mul_v8i32(<8 x i32> %arg1, <8 x i32> %arg2) #6 {
+    %ret = mul <8 x i32> %arg1, %arg2
+    ret <8 x i32> %ret
+  }
+
+  define <8 x i32> @test_mul_v8i32_avx512vl(<8 x i32> %arg1, <8 x i32> %arg2) #4 {
+    %ret = mul <8 x i32> %arg1, %arg2
+    ret <8 x i32> %ret
+  }
+
+  define <4 x i64> @test_mul_v4i64(<4 x i64> %arg1, <4 x i64> %arg2) #5 {
+    %ret = mul <4 x i64> %arg1, %arg2
+    ret <4 x i64> %ret
+  }
+
+  define <32 x i16> @test_mul_v32i16(<32 x i16> %arg1, <32 x i16> %arg2) #7 {
+    %ret = mul <32 x i16> %arg1, %arg2
+    ret <32 x i16> %ret
+  }
+
+  define <16 x i32> @test_mul_v16i32(<16 x i32> %arg1, <16 x i32> %arg2) #8 {
+    %ret = mul <16 x i32> %arg1, %arg2
+    ret <16 x i32> %ret
+  }
+
+  define <8 x i64> @test_mul_v8i64(<8 x i64> %arg1, <8 x i64> %arg2) #9 {
+    %ret = mul <8 x i64> %arg1, %arg2
+    ret <8 x i64> %ret
+  }
+
+  attributes #0 = { "target-features"="+sse2" }
+  attributes #1 = { "target-features"="+avx" }
+  attributes #2 = { "target-features"="+avx512vl,+avx512f,+avx512bw" }
+  attributes #3 = { "target-features"="+sse4.1" }
+  attributes #4 = { "target-features"="+avx512vl,+avx512f" }
+  attributes #5 = { "target-features"="+avx2,+avx512vl,+avx512f,+avx512dq" }
+  attributes #6 = { "target-features"="+avx2" }
+  attributes #7 = { "target-features"="+avx512f,+avx512bw" }
+  attributes #8 = { "target-features"="+avx512f" }
+  attributes #9 = { "target-features"="+avx512f,+avx512dq" }
+
+...
+---
+name:            test_mul_v8i16
+# CHECK-LABEL: name:  test_mul_v8i16
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128 }
+# CHECK-NEXT:   - { id: 1, class: vr128 }
+# CHECK-NEXT:   - { id: 2, class: vr128 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = PMULLWrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<8 x s16>) = COPY %xmm0
+    %1(<8 x s16>) = COPY %xmm1
+    %2(<8 x s16>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<8 x s16>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v8i16_avx
+# CHECK-LABEL: name:  test_mul_v8i16_avx
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128 }
+# CHECK-NEXT:   - { id: 1, class: vr128 }
+# CHECK-NEXT:   - { id: 2, class: vr128 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLWrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<8 x s16>) = COPY %xmm0
+    %1(<8 x s16>) = COPY %xmm1
+    %2(<8 x s16>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<8 x s16>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v8i16_avx512bwvl
+# CHECK-LABEL: name:  test_mul_v8i16_avx512bwvl
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128x }
+# CHECK-NEXT:   - { id: 1, class: vr128x }
+# CHECK-NEXT:   - { id: 2, class: vr128x }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLWZ128rr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<8 x s16>) = COPY %xmm0
+    %1(<8 x s16>) = COPY %xmm1
+    %2(<8 x s16>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<8 x s16>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v4i32
+# CHECK-LABEL: name:  test_mul_v4i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128 }
+# CHECK-NEXT:   - { id: 1, class: vr128 }
+# CHECK-NEXT:   - { id: 2, class: vr128 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = PMULLDrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<4 x s32>) = COPY %xmm0
+    %1(<4 x s32>) = COPY %xmm1
+    %2(<4 x s32>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<4 x s32>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v4i32_avx
+# CHECK-LABEL: name:  test_mul_v4i32_avx
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128 }
+# CHECK-NEXT:   - { id: 1, class: vr128 }
+# CHECK-NEXT:   - { id: 2, class: vr128 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLDrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<4 x s32>) = COPY %xmm0
+    %1(<4 x s32>) = COPY %xmm1
+    %2(<4 x s32>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<4 x s32>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v4i32_avx512vl
+# CHECK-LABEL: name:  test_mul_v4i32_avx512vl
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128x }
+# CHECK-NEXT:   - { id: 1, class: vr128x }
+# CHECK-NEXT:   - { id: 2, class: vr128x }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLDZ128rr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<4 x s32>) = COPY %xmm0
+    %1(<4 x s32>) = COPY %xmm1
+    %2(<4 x s32>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<4 x s32>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v2i64
+# CHECK-LABEL: name:  test_mul_v2i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr128x }
+# CHECK-NEXT:   - { id: 1, class: vr128x }
+# CHECK-NEXT:   - { id: 2, class: vr128x }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLQZ128rr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %xmm0, %xmm1
+
+    %0(<2 x s64>) = COPY %xmm0
+    %1(<2 x s64>) = COPY %xmm1
+    %2(<2 x s64>) = G_MUL %0, %1
+    %xmm0 = COPY %2(<2 x s64>)
+    RET 0, implicit %xmm0
+
+...
+---
+name:            test_mul_v16i16
+# CHECK-LABEL: name:  test_mul_v16i16
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr256 }
+# CHECK-NEXT:   - { id: 1, class: vr256 }
+# CHECK-NEXT:   - { id: 2, class: vr256 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLWYrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<16 x s16>) = COPY %ymm0
+    %1(<16 x s16>) = COPY %ymm1
+    %2(<16 x s16>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<16 x s16>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v16i16_avx512bwvl
+# CHECK-LABEL: name:  test_mul_v16i16_avx512bwvl
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr256x }
+# CHECK-NEXT:   - { id: 1, class: vr256x }
+# CHECK-NEXT:   - { id: 2, class: vr256x }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLWZ256rr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<16 x s16>) = COPY %ymm0
+    %1(<16 x s16>) = COPY %ymm1
+    %2(<16 x s16>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<16 x s16>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v8i32
+# CHECK-LABEL: name:  test_mul_v8i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr256 }
+# CHECK-NEXT:   - { id: 1, class: vr256 }
+# CHECK-NEXT:   - { id: 2, class: vr256 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLDYrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<8 x s32>) = COPY %ymm0
+    %1(<8 x s32>) = COPY %ymm1
+    %2(<8 x s32>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<8 x s32>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v8i32_avx512vl
+# CHECK-LABEL: name:  test_mul_v8i32_avx512vl
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr256x }
+# CHECK-NEXT:   - { id: 1, class: vr256x }
+# CHECK-NEXT:   - { id: 2, class: vr256x }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLDZ256rr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<8 x s32>) = COPY %ymm0
+    %1(<8 x s32>) = COPY %ymm1
+    %2(<8 x s32>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<8 x s32>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v4i64
+# CHECK-LABEL: name:  test_mul_v4i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr256x }
+# CHECK-NEXT:   - { id: 1, class: vr256x }
+# CHECK-NEXT:   - { id: 2, class: vr256x }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLQZ256rr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %ymm0, %ymm1
+
+    %0(<4 x s64>) = COPY %ymm0
+    %1(<4 x s64>) = COPY %ymm1
+    %2(<4 x s64>) = G_MUL %0, %1
+    %ymm0 = COPY %2(<4 x s64>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_mul_v32i16
+# CHECK-LABEL: name:  test_mul_v32i16
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr512 }
+# CHECK-NEXT:   - { id: 1, class: vr512 }
+# CHECK-NEXT:   - { id: 2, class: vr512 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLWZrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %zmm0, %zmm1
+
+    %0(<32 x s16>) = COPY %zmm0
+    %1(<32 x s16>) = COPY %zmm1
+    %2(<32 x s16>) = G_MUL %0, %1
+    %zmm0 = COPY %2(<32 x s16>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_mul_v16i32
+# CHECK-LABEL: name:  test_mul_v16i32
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr512 }
+# CHECK-NEXT:   - { id: 1, class: vr512 }
+# CHECK-NEXT:   - { id: 2, class: vr512 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLDZrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %zmm0, %zmm1
+
+    %0(<16 x s32>) = COPY %zmm0
+    %1(<16 x s32>) = COPY %zmm1
+    %2(<16 x s32>) = G_MUL %0, %1
+    %zmm0 = COPY %2(<16 x s32>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_mul_v8i64
+# CHECK-LABEL: name:  test_mul_v8i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+# CHECK:      registers:
+# CHECK-NEXT:   - { id: 0, class: vr512 }
+# CHECK-NEXT:   - { id: 1, class: vr512 }
+# CHECK-NEXT:   - { id: 2, class: vr512 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: vecr }
+  - { id: 2, class: vecr }
+# CHECK:          %2 = VPMULLQZrr %0, %1
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %zmm0, %zmm1
+
+    %0(<8 x s64>) = COPY %zmm0
+    %1(<8 x s64>) = COPY %zmm1
+    %2(<8 x s64>) = G_MUL %0, %1
+    %zmm0 = COPY %2(<8 x s64>)
+    RET 0, implicit %zmm0
+
+...
diff --git a/test/CodeGen/X86/addcarry.ll b/test/CodeGen/X86/addcarry.ll
index 5e95cd832789..be550e3fe2d1 100644
--- a/test/CodeGen/X86/addcarry.ll
+++ b/test/CodeGen/X86/addcarry.ll
@@ -204,3 +204,70 @@ entry:
   %6 = add i64 %4, %5
   ret i64 %6
 }
+
+%S = type { [4 x i64] }
+
+define %S @readd(%S* nocapture readonly %this, %S %arg.b) {
+; CHECK-LABEL: readd:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    addq (%rsi), %rdx
+; CHECK-NEXT:    movq 8(%rsi), %r10
+; CHECK-NEXT:    adcq $0, %r10
+; CHECK-NEXT:    sbbq %rax, %rax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    addq %rcx, %r10
+; CHECK-NEXT:    adcq 16(%rsi), %rax
+; CHECK-NEXT:    sbbq %rcx, %rcx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    addq %r8, %rax
+; CHECK-NEXT:    adcq 24(%rsi), %rcx
+; CHECK-NEXT:    addq %r9, %rcx
+; CHECK-NEXT:    movq %rdx, (%rdi)
+; CHECK-NEXT:    movq %r10, 8(%rdi)
+; CHECK-NEXT:    movq %rax, 16(%rdi)
+; CHECK-NEXT:    movq %rcx, 24(%rdi)
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    retq
+entry:
+  %0 = extractvalue %S %arg.b, 0
+  %.elt6 = extractvalue [4 x i64] %0, 1
+  %.elt8 = extractvalue [4 x i64] %0, 2
+  %.elt10 = extractvalue [4 x i64] %0, 3
+  %.elt = extractvalue [4 x i64] %0, 0
+  %1 = getelementptr inbounds %S, %S* %this, i64 0, i32 0, i64 0
+  %2 = load i64, i64* %1, align 8
+  %3 = zext i64 %2 to i128
+  %4 = zext i64 %.elt to i128
+  %5 = add nuw nsw i128 %3, %4
+  %6 = trunc i128 %5 to i64
+  %7 = lshr i128 %5, 64
+  %8 = getelementptr inbounds %S, %S* %this, i64 0, i32 0, i64 1
+  %9 = load i64, i64* %8, align 8
+  %10 = zext i64 %9 to i128
+  %11 = add nuw nsw i128 %7, %10
+  %12 = zext i64 %.elt6 to i128
+  %13 = add nuw nsw i128 %11, %12
+  %14 = trunc i128 %13 to i64
+  %15 = lshr i128 %13, 64
+  %16 = getelementptr inbounds %S, %S* %this, i64 0, i32 0, i64 2
+  %17 = load i64, i64* %16, align 8
+  %18 = zext i64 %17 to i128
+  %19 = add nuw nsw i128 %15, %18
+  %20 = zext i64 %.elt8 to i128
+  %21 = add nuw nsw i128 %19, %20
+  %22 = lshr i128 %21, 64
+  %23 = trunc i128 %21 to i64
+  %24 = getelementptr inbounds %S, %S* %this, i64 0,i32 0, i64 3
+  %25 = load i64, i64* %24, align 8
+  %26 = zext i64 %25 to i128
+  %27 = add nuw nsw i128 %22, %26
+  %28 = zext i64 %.elt10 to i128
+  %29 = add nuw nsw i128 %27, %28
+  %30 = trunc i128 %29 to i64
+  %31 = insertvalue [4 x i64] undef, i64 %6, 0
+  %32 = insertvalue [4 x i64] %31, i64 %14, 1
+  %33 = insertvalue [4 x i64] %32, i64 %23, 2
+  %34 = insertvalue [4 x i64] %33, i64 %30, 3
+  %35 = insertvalue %S undef, [4 x i64] %34, 0
+  ret %S %35
+}
diff --git a/test/CodeGen/X86/avx-isa-check.ll b/test/CodeGen/X86/avx-isa-check.ll
index dffc8078e44f..5d66dfde0bc6 100644
--- a/test/CodeGen/X86/avx-isa-check.ll
+++ b/test/CodeGen/X86/avx-isa-check.ll
@@ -680,3 +680,8 @@ define   <4 x double> @_inreg4xdouble(double %a) {
   %c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
   ret <4 x double> %c
 }
+
+define <8 x i32> @test_mul_v8i32(<8 x i32> %arg1, <8 x i32> %arg2) #0 {
+  %ret = mul <8 x i32> %arg1, %arg2
+  ret <8 x i32> %ret
+}
diff --git a/test/CodeGen/X86/avx1-logical-load-folding.ll b/test/CodeGen/X86/avx1-logical-load-folding.ll
index 90e00c965391..7073eb224763 100644
--- a/test/CodeGen/X86/avx1-logical-load-folding.ll
+++ b/test/CodeGen/X86/avx1-logical-load-folding.ll
@@ -1,10 +1,26 @@
-; RUN: llc -O3 -disable-peephole -mcpu=corei7-avx -mattr=+avx < %s | FileCheck %s
-
-target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.9.0"
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O3 -disable-peephole -mtriple=i686-apple-macosx10.9.0 -mcpu=corei7-avx -mattr=+avx | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -O3 -disable-peephole -mtriple=x86_64-apple-macosx10.9.0 -mcpu=corei7-avx -mattr=+avx | FileCheck %s --check-prefix=X64
 
 ; Function Attrs: nounwind ssp uwtable
 define void @test1(float* %A, float* %C) #0 {
+; X86-LABEL: test1:
+; X86:       ## BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovaps (%ecx), %ymm0
+; X86-NEXT:    vandps LCPI0_0, %ymm0, %ymm0
+; X86-NEXT:    vmovss %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; X64-LABEL: test1:
+; X64:       ## BB#0:
+; X64-NEXT:    vmovaps (%rdi), %ymm0
+; X64-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vmovss %xmm0, (%rsi)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
   %tmp1 = bitcast float* %A to <8 x float>*
   %tmp2 = load <8 x float>, <8 x float>* %tmp1, align 32
   %tmp3 = bitcast <8 x float> %tmp2 to <8 x i32>
@@ -13,12 +29,27 @@ define void @test1(float* %A, float* %C) #0 {
   %tmp6 = extractelement <8 x float> %tmp5, i32 0
   store float %tmp6, float* %C
   ret void
-
-  ; CHECK: vandps LCPI0_0(%rip), %ymm0, %ymm0
 }
 
 ; Function Attrs: nounwind ssp uwtable
 define void @test2(float* %A, float* %C) #0 {
+; X86-LABEL: test2:
+; X86:       ## BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovaps (%ecx), %ymm0
+; X86-NEXT:    vorps LCPI1_0, %ymm0, %ymm0
+; X86-NEXT:    vmovss %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; X64-LABEL: test2:
+; X64:       ## BB#0:
+; X64-NEXT:    vmovaps (%rdi), %ymm0
+; X64-NEXT:    vorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vmovss %xmm0, (%rsi)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
   %tmp1 = bitcast float* %A to <8 x float>*
   %tmp2 = load <8 x float>, <8 x float>* %tmp1, align 32
   %tmp3 = bitcast <8 x float> %tmp2 to <8 x i32>
@@ -27,12 +58,27 @@ define void @test2(float* %A, float* %C) #0 {
   %tmp6 = extractelement <8 x float> %tmp5, i32 0
   store float %tmp6, float* %C
   ret void
-
-  ; CHECK: vorps LCPI1_0(%rip), %ymm0, %ymm0
 }
 
 ; Function Attrs: nounwind ssp uwtable
 define void @test3(float* %A, float* %C) #0 {
+; X86-LABEL: test3:
+; X86:       ## BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovaps (%ecx), %ymm0
+; X86-NEXT:    vxorps LCPI2_0, %ymm0, %ymm0
+; X86-NEXT:    vmovss %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; X64-LABEL: test3:
+; X64:       ## BB#0:
+; X64-NEXT:    vmovaps (%rdi), %ymm0
+; X64-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vmovss %xmm0, (%rsi)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
   %tmp1 = bitcast float* %A to <8 x float>*
   %tmp2 = load <8 x float>, <8 x float>* %tmp1, align 32
   %tmp3 = bitcast <8 x float> %tmp2 to <8 x i32>
@@ -41,11 +87,26 @@ define void @test3(float* %A, float* %C) #0 {
   %tmp6 = extractelement <8 x float> %tmp5, i32 0
   store float %tmp6, float* %C
   ret void
-
-  ; CHECK: vxorps LCPI2_0(%rip), %ymm0, %ymm0
 }
 
 define void @test4(float* %A, float* %C) #0 {
+; X86-LABEL: test4:
+; X86:       ## BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovaps (%ecx), %ymm0
+; X86-NEXT:    vandnps LCPI3_0, %ymm0, %ymm0
+; X86-NEXT:    vmovss %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; X64-LABEL: test4:
+; X64:       ## BB#0:
+; X64-NEXT:    vmovaps (%rdi), %ymm0
+; X64-NEXT:    vandnps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vmovss %xmm0, (%rsi)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
   %tmp1 = bitcast float* %A to <8 x float>*
   %tmp2 = load <8 x float>, <8 x float>* %tmp1, align 32
   %tmp3 = bitcast <8 x float> %tmp2 to <8 x i32>
@@ -55,6 +116,4 @@ define void @test4(float* %A, float* %C) #0 {
   %tmp7 = extractelement <8 x float> %tmp6, i32 0
   store float %tmp7, float * %C
   ret void
-
-  ;CHECK: vandnps LCPI3_0(%rip), %ymm0, %ymm0
 }
diff --git a/test/CodeGen/X86/avx2-schedule.ll b/test/CodeGen/X86/avx2-schedule.ll
new file mode 100644
index 000000000000..042bc217b97c
--- /dev/null
+++ b/test/CodeGen/X86/avx2-schedule.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
+
+define <32 x i8> @test_pabsb(<32 x i8> %a0, <32 x i8> *%a1) {
+; HASWELL-LABEL: test_pabsb:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpabsb (%rdi), %ymm1 # sched: [5:0.50]
+; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pabsb:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpabsb (%rdi), %ymm1 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = call <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8> %a0)
+  %2 = load <32 x i8>, <32 x i8> *%a1, align 32
+  %3 = call <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8> %2)
+  %4 = or <32 x i8> %1, %3
+  ret <32 x i8> %4
+}
+declare <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8>) nounwind readnone
+
+define <8 x i32> @test_pabsd(<8 x i32> %a0, <8 x i32> *%a1) {
+; HASWELL-LABEL: test_pabsd:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpabsd (%rdi), %ymm1 # sched: [5:0.50]
+; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pabsd:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpabsd (%rdi), %ymm1 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = call <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32> %a0)
+  %2 = load <8 x i32>, <8 x i32> *%a1, align 32
+  %3 = call <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32> %2)
+  %4 = or <8 x i32> %1, %3
+  ret <8 x i32> %4
+}
+declare <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32>) nounwind readnone
+
+define <16 x i16> @test_pabsw(<16 x i16> %a0, <16 x i16> *%a1) {
+; HASWELL-LABEL: test_pabsw:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpabsw (%rdi), %ymm1 # sched: [5:0.50]
+; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pabsw:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpabsw (%rdi), %ymm1 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = call <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16> %a0)
+  %2 = load <16 x i16>, <16 x i16> *%a1, align 32
+  %3 = call <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16> %2)
+  %4 = or <16 x i16> %1, %3
+  ret <16 x i16> %4
+}
+declare <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16>) nounwind readnone
+
+define <32 x i8> @test_paddb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
+; HASWELL-LABEL: test_paddb:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_paddb:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = add <32 x i8> %a0, %a1
+  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
+  %3 = add <32 x i8> %1, %2
+  ret <32 x i8> %3
+}
+
+define <8 x i32> @test_paddd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
+; HASWELL-LABEL: test_paddd:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_paddd:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = add <8 x i32> %a0, %a1
+  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
+  %3 = add <8 x i32> %1, %2
+  ret <8 x i32> %3
+}
+
+define <4 x i64> @test_paddq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
+; HASWELL-LABEL: test_paddq:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_paddq:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = add <4 x i64> %a0, %a1
+  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
+  %3 = add <4 x i64> %1, %2
+  ret <4 x i64> %3
+}
+
+define <16 x i16> @test_paddw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
+; HASWELL-LABEL: test_paddw:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_paddw:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = add <16 x i16> %a0, %a1
+  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
+  %3 = add <16 x i16> %1, %2
+  ret <16 x i16> %3
+}
+
+define <4 x i64> @test_pand(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
+; HASWELL-LABEL: test_pand:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pand:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = and <4 x i64> %a0, %a1
+  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
+  %3 = and <4 x i64> %1, %2
+  %4 = add <4 x i64> %3, %a1
+  ret <4 x i64> %4
+}
+
+define <4 x i64> @test_pandn(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
+; HASWELL-LABEL: test_pandn:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [5:0.50]
+; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pandn:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = xor <4 x i64> %a0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %2 = and <4 x i64> %a1, %1
+  %3 = load <4 x i64>, <4 x i64> *%a2, align 32
+  %4 = xor <4 x i64> %2, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %5 = and <4 x i64> %3, %4
+  %6 = add <4 x i64> %2, %5
+  ret <4 x i64> %6
+}
+
+define <8 x i32> @test_pmulld(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
+; HASWELL-LABEL: test_pmulld:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [10:2.00]
+; HASWELL-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pmulld:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
+; ZNVER1-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = mul <8 x i32> %a0, %a1
+  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
+  %3 = mul <8 x i32> %1, %2
+  ret <8 x i32> %3
+}
+
+define <16 x i16> @test_pmullw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
+; HASWELL-LABEL: test_pmullw:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
+; HASWELL-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pmullw:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
+; ZNVER1-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = mul <16 x i16> %a0, %a1
+  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
+  %3 = mul <16 x i16> %1, %2
+  ret <16 x i16> %3
+}
+
+define <4 x i64> @test_por(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
+; HASWELL-LABEL: test_por:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_por:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = or <4 x i64> %a0, %a1
+  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
+  %3 = or <4 x i64> %1, %2
+  %4 = add <4 x i64> %3, %a1
+  ret <4 x i64> %4
+}
+
+define <32 x i8> @test_psubb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
+; HASWELL-LABEL: test_psubb:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_psubb:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = sub <32 x i8> %a0, %a1
+  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
+  %3 = sub <32 x i8> %1, %2
+  ret <32 x i8> %3
+}
+
+define <8 x i32> @test_psubd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
+; HASWELL-LABEL: test_psubd:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_psubd:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = sub <8 x i32> %a0, %a1
+  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
+  %3 = sub <8 x i32> %1, %2
+  ret <8 x i32> %3
+}
+
+define <4 x i64> @test_psubq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
+; HASWELL-LABEL: test_psubq:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_psubq:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = sub <4 x i64> %a0, %a1
+  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
+  %3 = sub <4 x i64> %1, %2
+  ret <4 x i64> %3
+}
+
+define <16 x i16> @test_psubw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
+; HASWELL-LABEL: test_psubw:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_psubw:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = sub <16 x i16> %a0, %a1
+  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
+  %3 = sub <16 x i16> %1, %2
+  ret <16 x i16> %3
+}
+
+define <4 x i64> @test_pxor(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
+; HASWELL-LABEL: test_pxor:
+; HASWELL:       # BB#0:
+; HASWELL-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
+; HASWELL-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [5:0.50]
+; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; HASWELL-NEXT:    retq # sched: [1:1.00]
+;
+; ZNVER1-LABEL: test_pxor:
+; ZNVER1:       # BB#0:
+; ZNVER1-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
+; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
+; ZNVER1-NEXT:    retq # sched: [4:1.00]
+  %1 = xor <4 x i64> %a0, %a1
+  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
+  %3 = xor <4 x i64> %1, %2
+  %4 = add <4 x i64> %3, %a1
+  ret <4 x i64> %4
+}
+
+!0 = !{i32 1}
diff --git a/test/CodeGen/X86/avx512vl-arith.ll b/test/CodeGen/X86/avx512vl-arith.ll
index ef01d8656dac..9c056cdee196 100644
--- a/test/CodeGen/X86/avx512vl-arith.ll
+++ b/test/CodeGen/X86/avx512vl-arith.ll
@@ -1,36 +1,42 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512vl| FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512vl --show-mc-encoding| FileCheck %s
 
 ; 256-bit
 
-; CHECK-LABEL: vpaddq256_test
-; CHECK: vpaddq %ymm{{.*}}
-; CHECK: ret
 define <4 x i64> @vpaddq256_test(<4 x i64> %i, <4 x i64> %j) nounwind readnone {
+; CHECK-LABEL: vpaddq256_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <4 x i64> %i, %j
   ret <4 x i64> %x
 }
 
-; CHECK-LABEL: vpaddq256_fold_test
-; CHECK: vpaddq (%rdi), %ymm{{.*}}
-; CHECK: ret
 define <4 x i64> @vpaddq256_fold_test(<4 x i64> %i, <4 x i64>* %j) nounwind {
+; CHECK-LABEL: vpaddq256_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load <4 x i64>, <4 x i64>* %j, align 4
   %x = add <4 x i64> %i, %tmp
   ret <4 x i64> %x
 }
 
-; CHECK-LABEL: vpaddq256_broadcast_test
-; CHECK: vpaddq LCP{{.*}}(%rip){1to4}, %ymm{{.*}}
-; CHECK: ret
 define <4 x i64> @vpaddq256_broadcast_test(<4 x i64> %i) nounwind {
+; CHECK-LABEL: vpaddq256_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq {{.*}}(%rip){1to4}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x38,0xd4,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI2_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <4 x i64> %i, <i64 1, i64 1, i64 1, i64 1>
   ret <4 x i64> %x
 }
 
-; CHECK-LABEL: vpaddq256_broadcast2_test
-; CHECK: vpaddq (%rdi){1to4}, %ymm{{.*}}
-; CHECK: ret
 define <4 x i64> @vpaddq256_broadcast2_test(<4 x i64> %i, i64* %j.ptr) nounwind {
+; CHECK-LABEL: vpaddq256_broadcast2_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq (%rdi){1to4}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x38,0xd4,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %j = load i64, i64* %j.ptr
   %j.0 = insertelement <4 x i64> undef, i64 %j, i32 0
   %j.v = shufflevector <4 x i64> %j.0, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -38,55 +44,68 @@ define <4 x i64> @vpaddq256_broadcast2_test(<4 x i64> %i, i64* %j.ptr) nounwind
   ret <4 x i64> %x
 }
 
-; CHECK-LABEL: vpaddd256_test
-; CHECK: vpaddd %ymm{{.*}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_test(<8 x i32> %i, <8 x i32> %j) nounwind readnone {
+; CHECK-LABEL: vpaddd256_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <8 x i32> %i, %j
   ret <8 x i32> %x
 }
 
-; CHECK-LABEL: vpaddd256_fold_test
-; CHECK: vpaddd (%rdi), %ymm{{.*}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_fold_test(<8 x i32> %i, <8 x i32>* %j) nounwind {
+; CHECK-LABEL: vpaddd256_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load <8 x i32>, <8 x i32>* %j, align 4
   %x = add <8 x i32> %i, %tmp
   ret <8 x i32> %x
 }
 
-; CHECK-LABEL: vpaddd256_broadcast_test
-; CHECK: vpaddd LCP{{.*}}(%rip){1to8}, %ymm{{.*}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_broadcast_test(<8 x i32> %i) nounwind {
+; CHECK-LABEL: vpaddd256_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddd {{.*}}(%rip){1to8}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x38,0xfe,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI6_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <8 x i32> %i, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i32> %x
 }
 
-; CHECK-LABEL: vpaddd256_mask_test
-; CHECK: vpaddd %ymm{{.*%k[1-7].*}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_mask_test(<8 x i32> %i, <8 x i32> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd256_mask_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm3, %ymm3, %ymm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe5,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqd %ymm3, %ymm2, %k1 ## encoding: [0x62,0xf3,0x6d,0x28,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfe,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = add <8 x i32> %i, %j
   %r = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %i
   ret <8 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd256_maskz_test
-; CHECK: vpaddd %ymm{{.*{%k[1-7]} {z}.*}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_maskz_test(<8 x i32> %i, <8 x i32> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd256_maskz_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm3, %ymm3, %ymm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe5,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqd %ymm3, %ymm2, %k1 ## encoding: [0x62,0xf3,0x6d,0x28,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfe,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = add <8 x i32> %i, %j
   %r = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
   ret <8 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd256_mask_fold_test
-; CHECK: vpaddd (%rdi), %ymm{{.*%k[1-7]}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_mask_fold_test(<8 x i32> %i, <8 x i32>* %j.ptr, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd256_mask_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm2, %ymm2, %ymm2 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf3,0x75,0x28,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xfe,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %j = load <8 x i32>, <8 x i32>* %j.ptr
   %x = add <8 x i32> %i, %j
@@ -94,20 +113,27 @@ define <8 x i32> @vpaddd256_mask_fold_test(<8 x i32> %i, <8 x i32>* %j.ptr, <8 x
   ret <8 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd256_mask_broadcast_test
-; CHECK: vpaddd LCP{{.*}}(%rip){1to8}, %ymm{{.*{%k[1-7]}}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_mask_broadcast_test(<8 x i32> %i, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd256_mask_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm2, %ymm2, %ymm2 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf3,0x75,0x28,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip){1to8}, %ymm0, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xfe,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI10_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = add <8 x i32> %i, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %r = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %i
   ret <8 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd256_maskz_fold_test
-; CHECK: vpaddd (%rdi), %ymm{{.*{%k[1-7]} {z}}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_maskz_fold_test(<8 x i32> %i, <8 x i32>* %j.ptr, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd256_maskz_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm2, %ymm2, %ymm2 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf3,0x75,0x28,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xfe,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %j = load <8 x i32>, <8 x i32>* %j.ptr
   %x = add <8 x i32> %i, %j
@@ -115,96 +141,111 @@ define <8 x i32> @vpaddd256_maskz_fold_test(<8 x i32> %i, <8 x i32>* %j.ptr, <8
   ret <8 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd256_maskz_broadcast_test
-; CHECK: vpaddd LCP{{.*}}(%rip){1to8}, %ymm{{.*{%k[1-7]} {z}}}
-; CHECK: ret
 define <8 x i32> @vpaddd256_maskz_broadcast_test(<8 x i32> %i, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd256_maskz_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm2, %ymm2, %ymm2 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf3,0x75,0x28,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xfe,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI12_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = add <8 x i32> %i, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %r = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
   ret <8 x i32> %r
 }
 
-; CHECK-LABEL: vpsubq256_test
-; CHECK: vpsubq %ymm{{.*}}
-; CHECK: ret
 define <4 x i64> @vpsubq256_test(<4 x i64> %i, <4 x i64> %j) nounwind readnone {
+; CHECK-LABEL: vpsubq256_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfb,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = sub <4 x i64> %i, %j
   ret <4 x i64> %x
 }
 
-; CHECK-LABEL: vpsubd256_test
-; CHECK: vpsubd %ymm{{.*}}
-; CHECK: ret
 define <8 x i32> @vpsubd256_test(<8 x i32> %i, <8 x i32> %j) nounwind readnone {
+; CHECK-LABEL: vpsubd256_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfa,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = sub <8 x i32> %i, %j
   ret <8 x i32> %x
 }
 
-; CHECK-LABEL: vpmulld256_test
-; CHECK: vpmulld %ymm{{.*}}
-; CHECK: ret
 define <8 x i32> @vpmulld256_test(<8 x i32> %i, <8 x i32> %j) {
+; CHECK-LABEL: vpmulld256_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x40,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = mul <8 x i32> %i, %j
   ret <8 x i32> %x
 }
 
-; CHECK-LABEL: test_vaddpd_256
-; CHECK: vaddpd{{.*}}
-; CHECK: ret
 define <4 x double> @test_vaddpd_256(<4 x double> %y, <4 x double> %x) {
+; CHECK-LABEL: test_vaddpd_256:
+; CHECK:       ## BB#0: ## %entry
+; CHECK-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf5,0x58,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %add.i = fadd <4 x double> %x, %y
   ret <4 x double> %add.i
 }
 
-; CHECK-LABEL: test_fold_vaddpd_256
-; CHECK: vaddpd LCP{{.*}}(%rip){{.*}}
-; CHECK: ret
 define <4 x double> @test_fold_vaddpd_256(<4 x double> %y) {
+; CHECK-LABEL: test_fold_vaddpd_256:
+; CHECK:       ## BB#0: ## %entry
+; CHECK-NEXT:    vaddpd {{.*}}(%rip), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x58,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 4, value: LCPI17_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %add.i = fadd <4 x double> %y, <double 4.500000e+00, double 3.400000e+00, double 4.500000e+00, double 5.600000e+00>
   ret <4 x double> %add.i
 }
 
-; CHECK-LABEL: test_broadcast_vaddpd_256
-; CHECK: LCP{{.*}}(%rip){1to8}, %ymm0, %ymm0
-; CHECK: ret
 define <8 x float> @test_broadcast_vaddpd_256(<8 x float> %a) nounwind {
+; CHECK-LABEL: test_broadcast_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vaddps {{.*}}(%rip){1to8}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7c,0x38,0x58,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI18_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = fadd <8 x float> %a, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
   ret <8 x float> %b
 }
 
-; CHECK-LABEL: test_mask_vaddps_256
-; CHECK: vaddps {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <8 x float> @test_mask_vaddps_256(<8 x float> %dst, <8 x float> %i,
-                                        <8 x float> %j, <8 x i32> %mask1)
-                                        nounwind readnone {
+define <8 x float> @test_mask_vaddps_256(<8 x float> %dst, <8 x float> %i, <8 x float> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vaddps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0x65,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vaddps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0x58,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = fadd <8 x float> %i, %j
   %r = select <8 x i1> %mask, <8 x float> %x, <8 x float> %dst
   ret <8 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vmulps_256
-; CHECK: vmulps {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <8 x float> @test_mask_vmulps_256(<8 x float> %dst, <8 x float> %i,
-                                        <8 x float> %j, <8 x i32> %mask1)
-                                        nounwind readnone {
+define <8 x float> @test_mask_vmulps_256(<8 x float> %dst, <8 x float> %i, <8 x float> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmulps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0x65,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmulps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0x59,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = fmul <8 x float> %i, %j
   %r = select <8 x i1> %mask, <8 x float> %x, <8 x float> %dst
   ret <8 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vminps_256
-; CHECK: vminps {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <8 x float> @test_mask_vminps_256(<8 x float> %dst, <8 x float> %i,
-                                        <8 x float> %j, <8 x i32> %mask1)
-                                        nounwind readnone {
+define <8 x float> @test_mask_vminps_256(<8 x float> %dst, <8 x float> %i, <8 x float> %j, <8 x i32> %mask1)nounwind readnone {
+; CHECK-LABEL: test_mask_vminps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0x65,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vminps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0x5d,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %cmp_res = fcmp olt <8 x float> %i, %j
   %min = select <8 x i1> %cmp_res, <8 x float> %i, <8 x float> %j
@@ -212,12 +253,13 @@ define <8 x float> @test_mask_vminps_256(<8 x float> %dst, <8 x float> %i,
   ret <8 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vmaxps_256
-; CHECK: vmaxps {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <8 x float> @test_mask_vmaxps_256(<8 x float> %dst, <8 x float> %i,
-                                        <8 x float> %j, <8 x i32> %mask1)
-                                        nounwind readnone {
+define <8 x float> @test_mask_vmaxps_256(<8 x float> %dst, <8 x float> %i, <8 x float> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmaxps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0x65,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmaxps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0x5f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %cmp_res = fcmp ogt <8 x float> %i, %j
   %max = select <8 x i1> %cmp_res, <8 x float> %i, <8 x float> %j
@@ -225,48 +267,52 @@ define <8 x float> @test_mask_vmaxps_256(<8 x float> %dst, <8 x float> %i,
   ret <8 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vsubps_256
-; CHECK: vsubps {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <8 x float> @test_mask_vsubps_256(<8 x float> %dst, <8 x float> %i,
-                                        <8 x float> %j, <8 x i32> %mask1)
-                                        nounwind readnone {
+define <8 x float> @test_mask_vsubps_256(<8 x float> %dst, <8 x float> %i, <8 x float> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vsubps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0x65,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vsubps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0x5c,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = fsub <8 x float> %i, %j
   %r = select <8 x i1> %mask, <8 x float> %x, <8 x float> %dst
   ret <8 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vdivps_256
-; CHECK: vdivps {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <8 x float> @test_mask_vdivps_256(<8 x float> %dst, <8 x float> %i,
-                                        <8 x float> %j, <8 x i32> %mask1)
-                                        nounwind readnone {
+define <8 x float> @test_mask_vdivps_256(<8 x float> %dst, <8 x float> %i, <8 x float> %j, <8 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vdivps_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0x65,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vdivps %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0x5e,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = fdiv <8 x float> %i, %j
   %r = select <8 x i1> %mask, <8 x float> %x, <8 x float> %dst
   ret <8 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vmulpd_256
-; CHECK: vmulpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x double> @test_mask_vmulpd_256(<4 x double> %dst, <4 x double> %i,
-                                        <4 x double> %j, <4 x i64> %mask1)
-                                        nounwind readnone {
+define <4 x double> @test_mask_vmulpd_256(<4 x double> %dst, <4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmulpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmulpd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x59,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %x = fmul <4 x double> %i, %j
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> %dst
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vminpd_256
-; CHECK: vminpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x double> @test_mask_vminpd_256(<4 x double> %dst, <4 x double> %i,
-                                        <4 x double> %j, <4 x i64> %mask1)
-                                        nounwind readnone {
+define <4 x double> @test_mask_vminpd_256(<4 x double> %dst, <4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vminpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vminpd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x5d,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %cmp_res = fcmp olt <4 x double> %i, %j
   %min = select <4 x i1> %cmp_res, <4 x double> %i, <4 x double> %j
@@ -274,12 +320,13 @@ define <4 x double> @test_mask_vminpd_256(<4 x double> %dst, <4 x double> %i,
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vmaxpd_256
-; CHECK: vmaxpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x double> @test_mask_vmaxpd_256(<4 x double> %dst, <4 x double> %i,
-                                        <4 x double> %j, <4 x i64> %mask1)
-                                        nounwind readnone {
+define <4 x double> @test_mask_vmaxpd_256(<4 x double> %dst, <4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmaxpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmaxpd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x5f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %cmp_res = fcmp ogt <4 x double> %i, %j
   %max = select <4 x i1> %cmp_res, <4 x double> %i, <4 x double> %j
@@ -287,59 +334,65 @@ define <4 x double> @test_mask_vmaxpd_256(<4 x double> %dst, <4 x double> %i,
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vsubpd_256
-; CHECK: vsubpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x double> @test_mask_vsubpd_256(<4 x double> %dst, <4 x double> %i,
-                                        <4 x double> %j, <4 x i64> %mask1)
-                                        nounwind readnone {
+define <4 x double> @test_mask_vsubpd_256(<4 x double> %dst, <4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vsubpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vsubpd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x5c,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %x = fsub <4 x double> %i, %j
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> %dst
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vdivpd_256
-; CHECK: vdivpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x double> @test_mask_vdivpd_256(<4 x double> %dst, <4 x double> %i,
-                                        <4 x double> %j, <4 x i64> %mask1)
-                                        nounwind readnone {
+define <4 x double> @test_mask_vdivpd_256(<4 x double> %dst, <4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vdivpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vdivpd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x5e,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %x = fdiv <4 x double> %i, %j
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> %dst
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vaddpd_256
-; CHECK: vaddpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x double> @test_mask_vaddpd_256(<4 x double> %dst, <4 x double> %i,
-                                         <4 x double> %j, <4 x i64> %mask1)
-                                         nounwind readnone {
+define <4 x double> @test_mask_vaddpd_256(<4 x double> %dst, <4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm4, %ymm4, %ymm4 ## EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %ymm4, %ymm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x28,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vaddpd %ymm2, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x58,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %x = fadd <4 x double> %i, %j
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> %dst
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_maskz_vaddpd_256
-; CHECK: vaddpd {{%ymm[0-9]{1,2}, %ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]} {z}}}
-; CHECK: ret
-define <4 x double> @test_maskz_vaddpd_256(<4 x double> %i, <4 x double> %j,
-                                          <4 x i64> %mask1) nounwind readnone {
+define <4 x double> @test_maskz_vaddpd_256(<4 x double> %i, <4 x double> %j, <4 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_maskz_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm3, %ymm3, %ymm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe5,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqq %ymm3, %ymm2, %k1 ## encoding: [0x62,0xf3,0xed,0x28,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0x58,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %x = fadd <4 x double> %i, %j
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> zeroinitializer
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_mask_fold_vaddpd_256
-; CHECK: vaddpd (%rdi), {{.*%ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]}.*}}
-; CHECK: ret
-define <4 x double> @test_mask_fold_vaddpd_256(<4 x double> %dst, <4 x double> %i,
-                                         <4 x double>* %j,  <4 x i64> %mask1)
-                                         nounwind {
+define <4 x double> @test_mask_fold_vaddpd_256(<4 x double> %dst, <4 x double> %i, <4 x double>* %j,  <4 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_mask_fold_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm3, %ymm3, %ymm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe5,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqq %ymm3, %ymm2, %k1 ## encoding: [0x62,0xf3,0xed,0x28,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vaddpd (%rdi), %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x29,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %tmp = load <4 x double>, <4 x double>* %j
   %x = fadd <4 x double> %i, %tmp
@@ -347,11 +400,13 @@ define <4 x double> @test_mask_fold_vaddpd_256(<4 x double> %dst, <4 x double> %
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_maskz_fold_vaddpd_256
-; CHECK: vaddpd (%rdi), {{.*%ymm[0-9]{1,2}, %ymm[0-9]{1,2} {%k[1-7]} {z}.*}}
-; CHECK: ret
-define <4 x double> @test_maskz_fold_vaddpd_256(<4 x double> %i, <4 x double>* %j,
-                                          <4 x i64> %mask1) nounwind {
+define <4 x double> @test_maskz_fold_vaddpd_256(<4 x double> %i, <4 x double>* %j, <4 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_maskz_fold_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm2, %ymm2, %ymm2 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqq %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf3,0xf5,0x28,0x1f,0xca,0x04]
+; CHECK-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xa9,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %tmp = load <4 x double>, <4 x double>* %j
   %x = fadd <4 x double> %i, %tmp
@@ -359,43 +414,46 @@ define <4 x double> @test_maskz_fold_vaddpd_256(<4 x double> %i, <4 x double>* %
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_broadcast2_vaddpd_256
-; CHECK: vaddpd (%rdi){1to4}, %ymm{{.*}}
-; CHECK: ret
 define <4 x double> @test_broadcast2_vaddpd_256(<4 x double> %i, double* %j) nounwind {
+; CHECK-LABEL: test_broadcast2_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vaddpd (%rdi){1to4}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x38,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load double, double* %j
   %b = insertelement <4 x double> undef, double %tmp, i32 0
-  %c = shufflevector <4 x double> %b, <4 x double> undef,
-                     <4 x i32> zeroinitializer
+  %c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
   %x = fadd <4 x double> %c, %i
   ret <4 x double> %x
 }
 
-; CHECK-LABEL: test_mask_broadcast_vaddpd_256
-; CHECK: vaddpd (%rdi){1to4}, %ymm{{.*{%k[1-7]}.*}}
-; CHECK: ret
-define <4 x double> @test_mask_broadcast_vaddpd_256(<4 x double> %dst, <4 x double> %i,
-                                          double* %j, <4 x i64> %mask1) nounwind {
+define <4 x double> @test_mask_broadcast_vaddpd_256(<4 x double> %dst, <4 x double> %i, double* %j, <4 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_mask_broadcast_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm0, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xef,0xc0]
+; CHECK-NEXT:    vpcmpneqq %ymm0, %ymm2, %k1 ## encoding: [0x62,0xf3,0xed,0x28,0x1f,0xc8,0x04]
+; CHECK-NEXT:    vaddpd (%rdi){1to4}, %ymm1, %ymm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x39,0x58,0x0f]
+; CHECK-NEXT:    vmovapd %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %tmp = load double, double* %j
   %b = insertelement <4 x double> undef, double %tmp, i32 0
-  %c = shufflevector <4 x double> %b, <4 x double> undef,
-                     <4 x i32> zeroinitializer
+  %c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
   %x = fadd <4 x double> %c, %i
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> %i
   ret <4 x double> %r
 }
 
-; CHECK-LABEL: test_maskz_broadcast_vaddpd_256
-; CHECK: vaddpd (%rdi){1to4}, %ymm{{.*{%k[1-7]} {z}.*}}
-; CHECK: ret
-define <4 x double> @test_maskz_broadcast_vaddpd_256(<4 x double> %i, double* %j,
-                                           <4 x i64> %mask1) nounwind {
+define <4 x double> @test_maskz_broadcast_vaddpd_256(<4 x double> %i, double* %j, <4 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_maskz_broadcast_vaddpd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %ymm2, %ymm2, %ymm2 ## EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqq %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf3,0xf5,0x28,0x1f,0xca,0x04]
+; CHECK-NEXT:    vaddpd (%rdi){1to4}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xb9,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i64> %mask1, zeroinitializer
   %tmp = load double, double* %j
   %b = insertelement <4 x double> undef, double %tmp, i32 0
-  %c = shufflevector <4 x double> %b, <4 x double> undef,
-                     <4 x i32> zeroinitializer
+  %c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
   %x = fadd <4 x double> %c, %i
   %r = select <4 x i1> %mask, <4 x double> %x, <4 x double> zeroinitializer
   ret <4 x double> %r
@@ -403,27 +461,30 @@ define <4 x double> @test_maskz_broadcast_vaddpd_256(<4 x double> %i, double* %j
 
 ; 128-bit
 
-; CHECK-LABEL: vpaddq128_test
-; CHECK: vpaddq %xmm{{.*}}
-; CHECK: ret
 define <2 x i64> @vpaddq128_test(<2 x i64> %i, <2 x i64> %j) nounwind readnone {
+; CHECK-LABEL: vpaddq128_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <2 x i64> %i, %j
   ret <2 x i64> %x
 }
 
-; CHECK-LABEL: vpaddq128_fold_test
-; CHECK: vpaddq (%rdi), %xmm{{.*}}
-; CHECK: ret
 define <2 x i64> @vpaddq128_fold_test(<2 x i64> %i, <2 x i64>* %j) nounwind {
+; CHECK-LABEL: vpaddq128_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load <2 x i64>, <2 x i64>* %j, align 4
   %x = add <2 x i64> %i, %tmp
   ret <2 x i64> %x
 }
 
-; CHECK-LABEL: vpaddq128_broadcast2_test
-; CHECK: vpaddq (%rdi){1to2}, %xmm{{.*}}
-; CHECK: ret
 define <2 x i64> @vpaddq128_broadcast2_test(<2 x i64> %i, i64* %j) nounwind {
+; CHECK-LABEL: vpaddq128_broadcast2_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddq (%rdi){1to2}, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x18,0xd4,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load i64, i64* %j
   %j.0 = insertelement <2 x i64> undef, i64 %tmp, i32 0
   %j.1 = insertelement <2 x i64> %j.0, i64 %tmp, i32 1
@@ -431,55 +492,68 @@ define <2 x i64> @vpaddq128_broadcast2_test(<2 x i64> %i, i64* %j) nounwind {
   ret <2 x i64> %x
 }
 
-; CHECK-LABEL: vpaddd128_test
-; CHECK: vpaddd %xmm{{.*}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_test(<4 x i32> %i, <4 x i32> %j) nounwind readnone {
+; CHECK-LABEL: vpaddd128_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <4 x i32> %i, %j
   ret <4 x i32> %x
 }
 
-; CHECK-LABEL: vpaddd128_fold_test
-; CHECK: vpaddd (%rdi), %xmm{{.*}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_fold_test(<4 x i32> %i, <4 x i32>* %j) nounwind {
+; CHECK-LABEL: vpaddd128_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load <4 x i32>, <4 x i32>* %j, align 4
   %x = add <4 x i32> %i, %tmp
   ret <4 x i32> %x
 }
 
-; CHECK-LABEL: vpaddd128_broadcast_test
-; CHECK: vpaddd LCP{{.*}}(%rip){1to4}, %xmm{{.*}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_broadcast_test(<4 x i32> %i) nounwind {
+; CHECK-LABEL: vpaddd128_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x18,0xfe,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI42_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <4 x i32> %i, <i32 1, i32 1, i32 1, i32 1>
   ret <4 x i32> %x
 }
 
-; CHECK-LABEL: vpaddd128_mask_test
-; CHECK: vpaddd %xmm{{.*%k[1-7].*}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_mask_test(<4 x i32> %i, <4 x i32> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd128_mask_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm3, %xmm3, %xmm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe1,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqd %xmm3, %xmm2, %k1 ## encoding: [0x62,0xf3,0x6d,0x08,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfe,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = add <4 x i32> %i, %j
   %r = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %i
   ret <4 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd128_maskz_test
-; CHECK: vpaddd %xmm{{.*{%k[1-7]} {z}.*}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_maskz_test(<4 x i32> %i, <4 x i32> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd128_maskz_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm3, %xmm3, %xmm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe1,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqd %xmm3, %xmm2, %k1 ## encoding: [0x62,0xf3,0x6d,0x08,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfe,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = add <4 x i32> %i, %j
   %r = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
   ret <4 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd128_mask_fold_test
-; CHECK: vpaddd (%rdi), %xmm{{.*%k[1-7]}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_mask_fold_test(<4 x i32> %i, <4 x i32>* %j.ptr, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd128_mask_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %xmm2, %xmm1, %k1 ## encoding: [0x62,0xf3,0x75,0x08,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xfe,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %j = load <4 x i32>, <4 x i32>* %j.ptr
   %x = add <4 x i32> %i, %j
@@ -487,20 +561,27 @@ define <4 x i32> @vpaddd128_mask_fold_test(<4 x i32> %i, <4 x i32>* %j.ptr, <4 x
   ret <4 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd128_mask_broadcast_test
-; CHECK: vpaddd LCP{{.*}}(%rip){1to4}, %xmm{{.*{%k[1-7]}}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_mask_broadcast_test(<4 x i32> %i, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd128_mask_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %xmm2, %xmm1, %k1 ## encoding: [0x62,0xf3,0x75,0x08,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xfe,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI46_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = add <4 x i32> %i, <i32 1, i32 1, i32 1, i32 1>
   %r = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %i
   ret <4 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd128_maskz_fold_test
-; CHECK: vpaddd (%rdi), %xmm{{.*{%k[1-7]} {z}}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_maskz_fold_test(<4 x i32> %i, <4 x i32>* %j.ptr, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd128_maskz_fold_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %xmm2, %xmm1, %k1 ## encoding: [0x62,0xf3,0x75,0x08,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xfe,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %j = load <4 x i32>, <4 x i32>* %j.ptr
   %x = add <4 x i32> %i, %j
@@ -508,96 +589,111 @@ define <4 x i32> @vpaddd128_maskz_fold_test(<4 x i32> %i, <4 x i32>* %j.ptr, <4
   ret <4 x i32> %r
 }
 
-; CHECK-LABEL: vpaddd128_maskz_broadcast_test
-; CHECK: vpaddd LCP{{.*}}(%rip){1to4}, %xmm{{.*{%k[1-7]} {z}}}
-; CHECK: ret
 define <4 x i32> @vpaddd128_maskz_broadcast_test(<4 x i32> %i, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: vpaddd128_maskz_broadcast_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqd %xmm2, %xmm1, %k1 ## encoding: [0x62,0xf3,0x75,0x08,0x1f,0xca,0x04]
+; CHECK-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xfe,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI48_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = add <4 x i32> %i, <i32 1, i32 1, i32 1, i32 1>
   %r = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
   ret <4 x i32> %r
 }
 
-; CHECK-LABEL: vpsubq128_test
-; CHECK: vpsubq %xmm{{.*}}
-; CHECK: ret
 define <2 x i64> @vpsubq128_test(<2 x i64> %i, <2 x i64> %j) nounwind readnone {
+; CHECK-LABEL: vpsubq128_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfb,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = sub <2 x i64> %i, %j
   ret <2 x i64> %x
 }
 
-; CHECK-LABEL: vpsubd128_test
-; CHECK: vpsubd %xmm{{.*}}
-; CHECK: ret
 define <4 x i32> @vpsubd128_test(<4 x i32> %i, <4 x i32> %j) nounwind readnone {
+; CHECK-LABEL: vpsubd128_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfa,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = sub <4 x i32> %i, %j
   ret <4 x i32> %x
 }
 
-; CHECK-LABEL: vpmulld128_test
-; CHECK: vpmulld %xmm{{.*}}
-; CHECK: ret
 define <4 x i32> @vpmulld128_test(<4 x i32> %i, <4 x i32> %j) {
+; CHECK-LABEL: vpmulld128_test:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x40,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = mul <4 x i32> %i, %j
   ret <4 x i32> %x
 }
 
-; CHECK-LABEL: test_vaddpd_128
-; CHECK: vaddpd{{.*}}
-; CHECK: ret
 define <2 x double> @test_vaddpd_128(<2 x double> %y, <2 x double> %x) {
+; CHECK-LABEL: test_vaddpd_128:
+; CHECK:       ## BB#0: ## %entry
+; CHECK-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0x58,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %add.i = fadd <2 x double> %x, %y
   ret <2 x double> %add.i
 }
 
-; CHECK-LABEL: test_fold_vaddpd_128
-; CHECK: vaddpd LCP{{.*}}(%rip){{.*}}
-; CHECK: ret
 define <2 x double> @test_fold_vaddpd_128(<2 x double> %y) {
+; CHECK-LABEL: test_fold_vaddpd_128:
+; CHECK:       ## BB#0: ## %entry
+; CHECK-NEXT:    vaddpd {{.*}}(%rip), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 4, value: LCPI53_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %add.i = fadd <2 x double> %y, <double 4.500000e+00, double 3.400000e+00>
   ret <2 x double> %add.i
 }
 
-; CHECK-LABEL: test_broadcast_vaddpd_128
-; CHECK: LCP{{.*}}(%rip){1to4}, %xmm0, %xmm0
-; CHECK: ret
 define <4 x float> @test_broadcast_vaddpd_128(<4 x float> %a) nounwind {
+; CHECK-LABEL: test_broadcast_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vaddps {{.*}}(%rip){1to4}, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7c,0x18,0x58,0x05,A,A,A,A]
+; CHECK-NEXT:    ## fixup A - offset: 6, value: LCPI54_0-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = fadd <4 x float> %a, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
   ret <4 x float> %b
 }
 
-; CHECK-LABEL: test_mask_vaddps_128
-; CHECK: vaddps {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x float> @test_mask_vaddps_128(<4 x float> %dst, <4 x float> %i,
-                                        <4 x float> %j, <4 x i32> %mask1)
-                                        nounwind readnone {
+define <4 x float> @test_mask_vaddps_128(<4 x float> %dst, <4 x float> %i, <4 x float> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vaddps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0x65,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vaddps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x09,0x58,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = fadd <4 x float> %i, %j
   %r = select <4 x i1> %mask, <4 x float> %x, <4 x float> %dst
   ret <4 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vmulps_128
-; CHECK: vmulps {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x float> @test_mask_vmulps_128(<4 x float> %dst, <4 x float> %i,
-                                        <4 x float> %j, <4 x i32> %mask1)
-                                        nounwind readnone {
+define <4 x float> @test_mask_vmulps_128(<4 x float> %dst, <4 x float> %i, <4 x float> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmulps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0x65,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmulps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x09,0x59,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = fmul <4 x float> %i, %j
   %r = select <4 x i1> %mask, <4 x float> %x, <4 x float> %dst
   ret <4 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vminps_128
-; CHECK: vminps {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x float> @test_mask_vminps_128(<4 x float> %dst, <4 x float> %i,
-                                        <4 x float> %j, <4 x i32> %mask1)
-                                        nounwind readnone {
+define <4 x float> @test_mask_vminps_128(<4 x float> %dst, <4 x float> %i, <4 x float> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vminps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0x65,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vminps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x09,0x5d,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %cmp_res = fcmp olt <4 x float> %i, %j
   %min = select <4 x i1> %cmp_res, <4 x float> %i, <4 x float> %j
@@ -605,12 +701,13 @@ define <4 x float> @test_mask_vminps_128(<4 x float> %dst, <4 x float> %i,
   ret <4 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vmaxps_128
-; CHECK: vmaxps {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x float> @test_mask_vmaxps_128(<4 x float> %dst, <4 x float> %i,
-                                        <4 x float> %j, <4 x i32> %mask1)
-                                        nounwind readnone {
+define <4 x float> @test_mask_vmaxps_128(<4 x float> %dst, <4 x float> %i, <4 x float> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmaxps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0x65,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmaxps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x09,0x5f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %cmp_res = fcmp ogt <4 x float> %i, %j
   %max = select <4 x i1> %cmp_res, <4 x float> %i, <4 x float> %j
@@ -618,12 +715,13 @@ define <4 x float> @test_mask_vmaxps_128(<4 x float> %dst, <4 x float> %i,
   ret <4 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vsubps_128
-; CHECK: vsubps {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x float> @test_mask_vsubps_128(<4 x float> %dst, <4 x float> %i,
-                                        <4 x float> %j, <4 x i32> %mask1)
-                                        nounwind readnone {
+define <4 x float> @test_mask_vsubps_128(<4 x float> %dst, <4 x float> %i, <4 x float> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vsubps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0x65,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vsubps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x09,0x5c,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = fsub <4 x float> %i, %j
   %r = select <4 x i1> %mask, <4 x float> %x, <4 x float> %dst
@@ -631,36 +729,39 @@ define <4 x float> @test_mask_vsubps_128(<4 x float> %dst, <4 x float> %i,
 }
 
 
-; CHECK-LABEL: test_mask_vdivps_128
-; CHECK: vdivps {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <4 x float> @test_mask_vdivps_128(<4 x float> %dst, <4 x float> %i,
-                                        <4 x float> %j, <4 x i32> %mask1)
-                                        nounwind readnone {
+define <4 x float> @test_mask_vdivps_128(<4 x float> %dst, <4 x float> %i, <4 x float> %j, <4 x i32> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vdivps_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqd %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0x65,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vdivps %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x74,0x09,0x5e,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = fdiv <4 x float> %i, %j
   %r = select <4 x i1> %mask, <4 x float> %x, <4 x float> %dst
   ret <4 x float> %r
 }
 
-; CHECK-LABEL: test_mask_vmulpd_128
-; CHECK: vmulpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <2 x double> @test_mask_vmulpd_128(<2 x double> %dst, <2 x double> %i,
-                                        <2 x double> %j, <2 x i64> %mask1)
-                                        nounwind readnone {
+define <2 x double> @test_mask_vmulpd_128(<2 x double> %dst, <2 x double> %i, <2 x double> %j, <2 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmulpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmulpd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x59,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %x = fmul <2 x double> %i, %j
   %r = select <2 x i1> %mask, <2 x double> %x, <2 x double> %dst
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vminpd_128
-; CHECK: vminpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <2 x double> @test_mask_vminpd_128(<2 x double> %dst, <2 x double> %i,
-                                        <2 x double> %j, <2 x i64> %mask1)
-                                        nounwind readnone {
+define <2 x double> @test_mask_vminpd_128(<2 x double> %dst, <2 x double> %i, <2 x double> %j, <2 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vminpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vminpd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x5d,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %cmp_res = fcmp olt <2 x double> %i, %j
   %min = select <2 x i1> %cmp_res, <2 x double> %i, <2 x double> %j
@@ -668,12 +769,13 @@ define <2 x double> @test_mask_vminpd_128(<2 x double> %dst, <2 x double> %i,
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vmaxpd_128
-; CHECK: vmaxpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <2 x double> @test_mask_vmaxpd_128(<2 x double> %dst, <2 x double> %i,
-                                        <2 x double> %j, <2 x i64> %mask1)
-                                        nounwind readnone {
+define <2 x double> @test_mask_vmaxpd_128(<2 x double> %dst, <2 x double> %i, <2 x double> %j, <2 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vmaxpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vmaxpd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x5f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %cmp_res = fcmp ogt <2 x double> %i, %j
   %max = select <2 x i1> %cmp_res, <2 x double> %i, <2 x double> %j
@@ -681,46 +783,52 @@ define <2 x double> @test_mask_vmaxpd_128(<2 x double> %dst, <2 x double> %i,
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vsubpd_128
-; CHECK: vsubpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <2 x double> @test_mask_vsubpd_128(<2 x double> %dst, <2 x double> %i,
-                                        <2 x double> %j, <2 x i64> %mask1)
-                                        nounwind readnone {
+define <2 x double> @test_mask_vsubpd_128(<2 x double> %dst, <2 x double> %i, <2 x double> %j, <2 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vsubpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vsubpd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x5c,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %x = fsub <2 x double> %i, %j
   %r = select <2 x i1> %mask, <2 x double> %x, <2 x double> %dst
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vdivpd_128
-; CHECK: vdivpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <2 x double> @test_mask_vdivpd_128(<2 x double> %dst, <2 x double> %i,
-                                        <2 x double> %j, <2 x i64> %mask1)
-                                        nounwind readnone {
+define <2 x double> @test_mask_vdivpd_128(<2 x double> %dst, <2 x double> %i, <2 x double> %j, <2 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vdivpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vdivpd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x5e,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %x = fdiv <2 x double> %i, %j
   %r = select <2 x i1> %mask, <2 x double> %x, <2 x double> %dst
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_mask_vaddpd_128
-; CHECK: vaddpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}}}
-; CHECK: ret
-define <2 x double> @test_mask_vaddpd_128(<2 x double> %dst, <2 x double> %i,
-                                         <2 x double> %j, <2 x i64> %mask1)
-                                         nounwind readnone {
+define <2 x double> @test_mask_vaddpd_128(<2 x double> %dst, <2 x double> %i, <2 x double> %j, <2 x i64> %mask1) nounwind readnone {
+; CHECK-LABEL: test_mask_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm4, %xmm4, %xmm4 ## EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xe4]
+; CHECK-NEXT:    vpcmpneqq %xmm4, %xmm3, %k1 ## encoding: [0x62,0xf3,0xe5,0x08,0x1f,0xcc,0x04]
+; CHECK-NEXT:    vaddpd %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x58,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %x = fadd <2 x double> %i, %j
   %r = select <2 x i1> %mask, <2 x double> %x, <2 x double> %dst
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_maskz_vaddpd_128
-; CHECK: vaddpd {{%xmm[0-9]{1,2}, %xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]} {z}}}
-; CHECK: ret
 define <2 x double> @test_maskz_vaddpd_128(<2 x double> %i, <2 x double> %j,
+; CHECK-LABEL: test_maskz_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm3, %xmm3, %xmm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe1,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqq %xmm3, %xmm2, %k1 ## encoding: [0x62,0xf3,0xed,0x08,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0x58,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
                                           <2 x i64> %mask1) nounwind readnone {
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %x = fadd <2 x double> %i, %j
@@ -728,12 +836,13 @@ define <2 x double> @test_maskz_vaddpd_128(<2 x double> %i, <2 x double> %j,
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_mask_fold_vaddpd_128
-; CHECK: vaddpd (%rdi), {{.*%xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]}.*}}
-; CHECK: ret
-define <2 x double> @test_mask_fold_vaddpd_128(<2 x double> %dst, <2 x double> %i,
-                                         <2 x double>* %j,  <2 x i64> %mask1)
-                                         nounwind {
+define <2 x double> @test_mask_fold_vaddpd_128(<2 x double> %dst, <2 x double> %i, <2 x double>* %j,  <2 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_mask_fold_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm3, %xmm3, %xmm3 ## EVEX TO VEX Compression encoding: [0xc5,0xe1,0xef,0xdb]
+; CHECK-NEXT:    vpcmpneqq %xmm3, %xmm2, %k1 ## encoding: [0x62,0xf3,0xed,0x08,0x1f,0xcb,0x04]
+; CHECK-NEXT:    vaddpd (%rdi), %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf1,0xf5,0x09,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %tmp = load <2 x double>, <2 x double>* %j
   %x = fadd <2 x double> %i, %tmp
@@ -741,11 +850,13 @@ define <2 x double> @test_mask_fold_vaddpd_128(<2 x double> %dst, <2 x double> %
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_maskz_fold_vaddpd_128
-; CHECK: vaddpd (%rdi), {{.*%xmm[0-9]{1,2}, %xmm[0-9]{1,2} {%k[1-7]} {z}.*}}
-; CHECK: ret
-define <2 x double> @test_maskz_fold_vaddpd_128(<2 x double> %i, <2 x double>* %j,
-                                          <2 x i64> %mask1) nounwind {
+define <2 x double> @test_maskz_fold_vaddpd_128(<2 x double> %i, <2 x double>* %j, <2 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_maskz_fold_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqq %xmm2, %xmm1, %k1 ## encoding: [0x62,0xf3,0xf5,0x08,0x1f,0xca,0x04]
+; CHECK-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x89,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %tmp = load <2 x double>, <2 x double>* %j
   %x = fadd <2 x double> %i, %tmp
@@ -753,10 +864,11 @@ define <2 x double> @test_maskz_fold_vaddpd_128(<2 x double> %i, <2 x double>* %
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_broadcast2_vaddpd_128
-; CHECK: vaddpd (%rdi){1to2}, %xmm{{.*}}
-; CHECK: ret
 define <2 x double> @test_broadcast2_vaddpd_128(<2 x double> %i, double* %j) nounwind {
+; CHECK-LABEL: test_broadcast2_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vaddpd (%rdi){1to2}, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x18,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %tmp = load double, double* %j
   %j.0 = insertelement <2 x double> undef, double %tmp, i64 0
   %j.1 = insertelement <2 x double> %j.0, double %tmp, i64 1
@@ -764,12 +876,14 @@ define <2 x double> @test_broadcast2_vaddpd_128(<2 x double> %i, double* %j) nou
   ret <2 x double> %x
 }
 
-; CHECK-LABEL: test_mask_broadcast_vaddpd_128
-; CHECK: vaddpd (%rdi){1to2}, %xmm{{.*{%k[1-7]}.*}}
-; CHECK: ret
-define <2 x double> @test_mask_broadcast_vaddpd_128(<2 x double> %dst, <2 x double> %i,
-                                          double* %j, <2 x i64> %mask1)
-                                          nounwind {
+define <2 x double> @test_mask_broadcast_vaddpd_128(<2 x double> %dst, <2 x double> %i, double* %j, <2 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_mask_broadcast_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xef,0xc0]
+; CHECK-NEXT:    vpcmpneqq %xmm0, %xmm2, %k1 ## encoding: [0x62,0xf3,0xed,0x08,0x1f,0xc8,0x04]
+; CHECK-NEXT:    vaddpd (%rdi){1to2}, %xmm1, %xmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x19,0x58,0x0f]
+; CHECK-NEXT:    vmovapd %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %tmp = load double, double* %j
   %j.0 = insertelement <2 x double> undef, double %tmp, i64 0
@@ -779,11 +893,13 @@ define <2 x double> @test_mask_broadcast_vaddpd_128(<2 x double> %dst, <2 x doub
   ret <2 x double> %r
 }
 
-; CHECK-LABEL: test_maskz_broadcast_vaddpd_128
-; CHECK: vaddpd (%rdi){1to2}, %xmm{{.*{%k[1-7]} {z}.*}}
-; CHECK: ret
-define <2 x double> @test_maskz_broadcast_vaddpd_128(<2 x double> %i, double* %j,
-                                           <2 x i64> %mask1) nounwind {
+define <2 x double> @test_maskz_broadcast_vaddpd_128(<2 x double> %i, double* %j, <2 x i64> %mask1) nounwind {
+; CHECK-LABEL: test_maskz_broadcast_vaddpd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; CHECK-NEXT:    vpcmpneqq %xmm2, %xmm1, %k1 ## encoding: [0x62,0xf3,0xf5,0x08,0x1f,0xca,0x04]
+; CHECK-NEXT:    vaddpd (%rdi){1to2}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x99,0x58,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <2 x i64> %mask1, zeroinitializer
   %tmp = load double, double* %j
   %j.0 = insertelement <2 x double> undef, double %tmp, i64 0
diff --git a/test/CodeGen/X86/branchfolding-undef.mir b/test/CodeGen/X86/branchfolding-undef.mir
index 0da167b33257..1a7dfb941875 100644
--- a/test/CodeGen/X86/branchfolding-undef.mir
+++ b/test/CodeGen/X86/branchfolding-undef.mir
@@ -16,7 +16,6 @@ name: func
 tracksRegLiveness: true
 body: |
   bb.0:
-    successors: %bb.1, %bb.2
     JE_1 %bb.1, implicit undef %eflags
     JMP_1 %bb.2
 
diff --git a/test/CodeGen/X86/build-vector-128.ll b/test/CodeGen/X86/build-vector-128.ll
new file mode 100644
index 000000000000..8c3a6790ffa6
--- /dev/null
+++ b/test/CodeGen/X86/build-vector-128.ll
@@ -0,0 +1,428 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE-32 --check-prefix=SSE2-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE-64 --check-prefix=SSE2-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE-32 --check-prefix=SSE41-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE-64 --check-prefix=SSE41-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX-32 --check-prefix=AVX1-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX-64 --check-prefix=AVX1-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX-32 --check-prefix=AVX2-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX-64 --check-prefix=AVX2-64
+
+define <2 x double> @test_buildvector_v2f64(double %a0, double %a1) {
+; SSE-32-LABEL: test_buildvector_v2f64:
+; SSE-32:       # BB#0:
+; SSE-32-NEXT:    movups {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_buildvector_v2f64:
+; SSE-64:       # BB#0:
+; SSE-64-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_buildvector_v2f64:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v2f64:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <2 x double> undef, double %a0, i32 0
+  %ins1 = insertelement <2 x double> %ins0, double %a1, i32 1
+  ret <2 x double> %ins1
+}
+
+define <4 x float> @test_buildvector_v4f32(float %a0, float %a1, float %a2, float %a3) {
+; SSE-32-LABEL: test_buildvector_v4f32:
+; SSE-32:       # BB#0:
+; SSE-32-NEXT:    movups {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE2-64-LABEL: test_buildvector_v4f32:
+; SSE2-64:       # BB#0:
+; SSE2-64-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-64-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-64-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-64-NEXT:    retq
+;
+; SSE41-64-LABEL: test_buildvector_v4f32:
+; SSE41-64:       # BB#0:
+; SSE41-64-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SSE41-64-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; SSE41-64-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_buildvector_v4f32:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v4f32:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <4 x float> undef, float %a0, i32 0
+  %ins1 = insertelement <4 x float> %ins0, float %a1, i32 1
+  %ins2 = insertelement <4 x float> %ins1, float %a2, i32 2
+  %ins3 = insertelement <4 x float> %ins2, float %a3, i32 3
+  ret <4 x float> %ins3
+}
+
+define <2 x i64> @test_buildvector_v2i64(i64 %a0, i64 %a1) {
+; SSE2-32-LABEL: test_buildvector_v2i64:
+; SSE2-32:       # BB#0:
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-32-NEXT:    retl
+;
+; SSE-64-LABEL: test_buildvector_v2i64:
+; SSE-64:       # BB#0:
+; SSE-64-NEXT:    movq %rsi, %xmm1
+; SSE-64-NEXT:    movq %rdi, %xmm0
+; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-64-NEXT:    retq
+;
+; SSE41-32-LABEL: test_buildvector_v2i64:
+; SSE41-32:       # BB#0:
+; SSE41-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    pinsrd $1, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrd $2, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrd $3, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    retl
+;
+; AVX-32-LABEL: test_buildvector_v2i64:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v2i64:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vmovq %rsi, %xmm0
+; AVX-64-NEXT:    vmovq %rdi, %xmm1
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <2 x i64> undef, i64 %a0, i32 0
+  %ins1 = insertelement <2 x i64> %ins0, i64 %a1, i32 1
+  ret <2 x i64> %ins1
+}
+
+define <4 x i32> @test_buildvector_v4i32(i32 %f0, i32 %f1, i32 %f2, i32 %f3) {
+; SSE-32-LABEL: test_buildvector_v4i32:
+; SSE-32:       # BB#0:
+; SSE-32-NEXT:    movups {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    retl
+;
+; SSE2-64-LABEL: test_buildvector_v4i32:
+; SSE2-64:       # BB#0:
+; SSE2-64-NEXT:    movd %ecx, %xmm0
+; SSE2-64-NEXT:    movd %esi, %xmm1
+; SSE2-64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-64-NEXT:    movd %edx, %xmm2
+; SSE2-64-NEXT:    movd %edi, %xmm0
+; SSE2-64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-64-NEXT:    retq
+;
+; SSE41-64-LABEL: test_buildvector_v4i32:
+; SSE41-64:       # BB#0:
+; SSE41-64-NEXT:    movd %edi, %xmm0
+; SSE41-64-NEXT:    pinsrd $1, %esi, %xmm0
+; SSE41-64-NEXT:    pinsrd $2, %edx, %xmm0
+; SSE41-64-NEXT:    pinsrd $3, %ecx, %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_buildvector_v4i32:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v4i32:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vmovd %edi, %xmm0
+; AVX-64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <4 x i32> undef, i32 %f0, i32 0
+  %ins1 = insertelement <4 x i32> %ins0, i32 %f1, i32 1
+  %ins2 = insertelement <4 x i32> %ins1, i32 %f2, i32 2
+  %ins3 = insertelement <4 x i32> %ins2, i32 %f3, i32 3
+  ret <4 x i32> %ins3
+}
+
+define <8 x i16> @test_buildvector_v8i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7) {
+; SSE2-32-LABEL: test_buildvector_v8i16:
+; SSE2-32:       # BB#0:
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-32-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-32-NEXT:    retl
+;
+; SSE2-64-LABEL: test_buildvector_v8i16:
+; SSE2-64:       # BB#0:
+; SSE2-64-NEXT:    movd %ecx, %xmm0
+; SSE2-64-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-64-NEXT:    movd %r9d, %xmm1
+; SSE2-64-NEXT:    movd %esi, %xmm2
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-64-NEXT:    movd %edx, %xmm1
+; SSE2-64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-64-NEXT:    movd %r8d, %xmm3
+; SSE2-64-NEXT:    movd %edi, %xmm0
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-64-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-64-NEXT:    retq
+;
+; SSE41-32-LABEL: test_buildvector_v8i16:
+; SSE41-32:       # BB#0:
+; SSE41-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    pinsrw $1, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrw $2, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrw $3, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrw $4, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrw $5, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrw $6, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrw $7, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: test_buildvector_v8i16:
+; SSE41-64:       # BB#0:
+; SSE41-64-NEXT:    movd %edi, %xmm0
+; SSE41-64-NEXT:    pinsrw $1, %esi, %xmm0
+; SSE41-64-NEXT:    pinsrw $2, %edx, %xmm0
+; SSE41-64-NEXT:    pinsrw $3, %ecx, %xmm0
+; SSE41-64-NEXT:    pinsrw $4, %r8d, %xmm0
+; SSE41-64-NEXT:    pinsrw $5, %r9d, %xmm0
+; SSE41-64-NEXT:    pinsrw $6, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrw $7, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_buildvector_v8i16:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v8i16:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vmovd %edi, %xmm0
+; AVX-64-NEXT:    vpinsrw $1, %esi, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $2, %edx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $4, %r8d, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $5, %r9d, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <8 x i16> undef, i16 %a0, i32 0
+  %ins1 = insertelement <8 x i16> %ins0, i16 %a1, i32 1
+  %ins2 = insertelement <8 x i16> %ins1, i16 %a2, i32 2
+  %ins3 = insertelement <8 x i16> %ins2, i16 %a3, i32 3
+  %ins4 = insertelement <8 x i16> %ins3, i16 %a4, i32 4
+  %ins5 = insertelement <8 x i16> %ins4, i16 %a5, i32 5
+  %ins6 = insertelement <8 x i16> %ins5, i16 %a6, i32 6
+  %ins7 = insertelement <8 x i16> %ins6, i16 %a7, i32 7
+  ret <8 x i16> %ins7
+}
+
+define <16 x i8> @test_buildvector_v16i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15) {
+; SSE2-32-LABEL: test_buildvector_v16i8:
+; SSE2-32:       # BB#0:
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-32-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-32-NEXT:    retl
+;
+; SSE2-64-LABEL: test_buildvector_v16i8:
+; SSE2-64:       # BB#0:
+; SSE2-64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-64-NEXT:    movd %ecx, %xmm0
+; SSE2-64-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-64-NEXT:    movd %r9d, %xmm1
+; SSE2-64-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-64-NEXT:    movd %esi, %xmm2
+; SSE2-64-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-64-NEXT:    movd %edx, %xmm3
+; SSE2-64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-64-NEXT:    movd %r8d, %xmm1
+; SSE2-64-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-64-NEXT:    movd %edi, %xmm0
+; SSE2-64-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-64-NEXT:    retq
+;
+; SSE41-32-LABEL: test_buildvector_v16i8:
+; SSE41-32:       # BB#0:
+; SSE41-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-32-NEXT:    pinsrb $1, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $2, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $3, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $4, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $5, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $6, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $7, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $8, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $9, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $10, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $11, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $12, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $13, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $14, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    pinsrb $15, {{[0-9]+}}(%esp), %xmm0
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: test_buildvector_v16i8:
+; SSE41-64:       # BB#0:
+; SSE41-64-NEXT:    movd %edi, %xmm0
+; SSE41-64-NEXT:    pinsrb $1, %esi, %xmm0
+; SSE41-64-NEXT:    pinsrb $2, %edx, %xmm0
+; SSE41-64-NEXT:    pinsrb $3, %ecx, %xmm0
+; SSE41-64-NEXT:    pinsrb $4, %r8d, %xmm0
+; SSE41-64-NEXT:    pinsrb $5, %r9d, %xmm0
+; SSE41-64-NEXT:    pinsrb $6, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $7, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $8, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $9, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $10, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $11, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $12, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $13, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $14, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    pinsrb $15, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: test_buildvector_v16i8:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v16i8:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vmovd %edi, %xmm0
+; AVX-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX-64-NEXT:    retq
+  %ins0  = insertelement <16 x i8> undef,  i8 %a0,  i32 0
+  %ins1  = insertelement <16 x i8> %ins0,  i8 %a1,  i32 1
+  %ins2  = insertelement <16 x i8> %ins1,  i8 %a2,  i32 2
+  %ins3  = insertelement <16 x i8> %ins2,  i8 %a3,  i32 3
+  %ins4  = insertelement <16 x i8> %ins3,  i8 %a4,  i32 4
+  %ins5  = insertelement <16 x i8> %ins4,  i8 %a5,  i32 5
+  %ins6  = insertelement <16 x i8> %ins5,  i8 %a6,  i32 6
+  %ins7  = insertelement <16 x i8> %ins6,  i8 %a7,  i32 7
+  %ins8  = insertelement <16 x i8> %ins7,  i8 %a8,  i32 8
+  %ins9  = insertelement <16 x i8> %ins8,  i8 %a9,  i32 9
+  %ins10 = insertelement <16 x i8> %ins9,  i8 %a10, i32 10
+  %ins11 = insertelement <16 x i8> %ins10, i8 %a11, i32 11
+  %ins12 = insertelement <16 x i8> %ins11, i8 %a12, i32 12
+  %ins13 = insertelement <16 x i8> %ins12, i8 %a13, i32 13
+  %ins14 = insertelement <16 x i8> %ins13, i8 %a14, i32 14
+  %ins15 = insertelement <16 x i8> %ins14, i8 %a15, i32 15
+  ret <16 x i8> %ins15
+}
diff --git a/test/CodeGen/X86/build-vector-256.ll b/test/CodeGen/X86/build-vector-256.ll
new file mode 100644
index 000000000000..1ced1fc3a382
--- /dev/null
+++ b/test/CodeGen/X86/build-vector-256.ll
@@ -0,0 +1,434 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX-32 --check-prefix=AVX1-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX-64 --check-prefix=AVX1-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX-32 --check-prefix=AVX2-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX-64 --check-prefix=AVX2-64
+
+define <4 x double> @test_buildvector_v4f64(double %a0, double %a1, double %a2, double %a3) {
+; AVX-32-LABEL: test_buildvector_v4f64:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %ymm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v4f64:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <4 x double> undef, double %a0, i32 0
+  %ins1 = insertelement <4 x double> %ins0, double %a1, i32 1
+  %ins2 = insertelement <4 x double> %ins1, double %a2, i32 2
+  %ins3 = insertelement <4 x double> %ins2, double %a3, i32 3
+  ret <4 x double> %ins3
+}
+
+define <8 x float> @test_buildvector_v8f32(float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7) {
+; AVX-32-LABEL: test_buildvector_v8f32:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %ymm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v8f32:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <8 x float> undef, float %a0, i32 0
+  %ins1 = insertelement <8 x float> %ins0, float %a1, i32 1
+  %ins2 = insertelement <8 x float> %ins1, float %a2, i32 2
+  %ins3 = insertelement <8 x float> %ins2, float %a3, i32 3
+  %ins4 = insertelement <8 x float> %ins3, float %a4, i32 4
+  %ins5 = insertelement <8 x float> %ins4, float %a5, i32 5
+  %ins6 = insertelement <8 x float> %ins5, float %a6, i32 6
+  %ins7 = insertelement <8 x float> %ins6, float %a7, i32 7
+  ret <8 x float> %ins7
+}
+
+define <4 x i64> @test_buildvector_v4i64(i64 %a0, i64 %a1, i64 %a2, i64 %a3) {
+; AVX1-32-LABEL: test_buildvector_v4i64:
+; AVX1-32:       # BB#0:
+; AVX1-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: test_buildvector_v4i64:
+; AVX1-64:       # BB#0:
+; AVX1-64-NEXT:    vmovq %rcx, %xmm0
+; AVX1-64-NEXT:    vmovq %rdx, %xmm1
+; AVX1-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-64-NEXT:    vmovq %rsi, %xmm1
+; AVX1-64-NEXT:    vmovq %rdi, %xmm2
+; AVX1-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-32-LABEL: test_buildvector_v4i64:
+; AVX2-32:       # BB#0:
+; AVX2-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-32-NEXT:    retl
+;
+; AVX2-64-LABEL: test_buildvector_v4i64:
+; AVX2-64:       # BB#0:
+; AVX2-64-NEXT:    vmovq %rcx, %xmm0
+; AVX2-64-NEXT:    vmovq %rdx, %xmm1
+; AVX2-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-64-NEXT:    vmovq %rsi, %xmm1
+; AVX2-64-NEXT:    vmovq %rdi, %xmm2
+; AVX2-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-64-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-64-NEXT:    retq
+  %ins0 = insertelement <4 x i64> undef, i64 %a0, i32 0
+  %ins1 = insertelement <4 x i64> %ins0, i64 %a1, i32 1
+  %ins2 = insertelement <4 x i64> %ins1, i64 %a2, i32 2
+  %ins3 = insertelement <4 x i64> %ins2, i64 %a3, i32 3
+  ret <4 x i64> %ins3
+}
+
+define <8 x i32> @test_buildvector_v8i32(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7) {
+; AVX-32-LABEL: test_buildvector_v8i32:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %ymm0
+; AVX-32-NEXT:    retl
+;
+; AVX1-64-LABEL: test_buildvector_v8i32:
+; AVX1-64:       # BB#0:
+; AVX1-64-NEXT:    vmovd %edi, %xmm0
+; AVX1-64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX1-64-NEXT:    vmovd %r8d, %xmm1
+; AVX1-64-NEXT:    vpinsrd $1, %r9d, %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-64-LABEL: test_buildvector_v8i32:
+; AVX2-64:       # BB#0:
+; AVX2-64-NEXT:    vmovd %edi, %xmm0
+; AVX2-64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX2-64-NEXT:    vmovd %r8d, %xmm1
+; AVX2-64-NEXT:    vpinsrd $1, %r9d, %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-64-NEXT:    retq
+  %ins0 = insertelement <8 x i32> undef, i32 %a0, i32 0
+  %ins1 = insertelement <8 x i32> %ins0, i32 %a1, i32 1
+  %ins2 = insertelement <8 x i32> %ins1, i32 %a2, i32 2
+  %ins3 = insertelement <8 x i32> %ins2, i32 %a3, i32 3
+  %ins4 = insertelement <8 x i32> %ins3, i32 %a4, i32 4
+  %ins5 = insertelement <8 x i32> %ins4, i32 %a5, i32 5
+  %ins6 = insertelement <8 x i32> %ins5, i32 %a6, i32 6
+  %ins7 = insertelement <8 x i32> %ins6, i32 %a7, i32 7
+  ret <8 x i32> %ins7
+}
+
+define <16 x i16> @test_buildvector_v16i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7, i16 %a8, i16 %a9, i16 %a10, i16 %a11, i16 %a12, i16 %a13, i16 %a14, i16 %a15) {
+; AVX1-32-LABEL: test_buildvector_v16i16:
+; AVX1-32:       # BB#0:
+; AVX1-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: test_buildvector_v16i16:
+; AVX1-64:       # BB#0:
+; AVX1-64-NEXT:    vmovd %edi, %xmm0
+; AVX1-64-NEXT:    vpinsrw $1, %esi, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrw $2, %edx, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrw $4, %r8d, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrw $5, %r9d, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-32-LABEL: test_buildvector_v16i16:
+; AVX2-32:       # BB#0:
+; AVX2-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-32-NEXT:    retl
+;
+; AVX2-64-LABEL: test_buildvector_v16i16:
+; AVX2-64:       # BB#0:
+; AVX2-64-NEXT:    vmovd %edi, %xmm0
+; AVX2-64-NEXT:    vpinsrw $1, %esi, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrw $2, %edx, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrw $4, %r8d, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrw $5, %r9d, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-64-NEXT:    retq
+  %ins0  = insertelement <16 x i16> undef,  i16 %a0,  i32 0
+  %ins1  = insertelement <16 x i16> %ins0,  i16 %a1,  i32 1
+  %ins2  = insertelement <16 x i16> %ins1,  i16 %a2,  i32 2
+  %ins3  = insertelement <16 x i16> %ins2,  i16 %a3,  i32 3
+  %ins4  = insertelement <16 x i16> %ins3,  i16 %a4,  i32 4
+  %ins5  = insertelement <16 x i16> %ins4,  i16 %a5,  i32 5
+  %ins6  = insertelement <16 x i16> %ins5,  i16 %a6,  i32 6
+  %ins7  = insertelement <16 x i16> %ins6,  i16 %a7,  i32 7
+  %ins8  = insertelement <16 x i16> %ins7,  i16 %a8,  i32 8
+  %ins9  = insertelement <16 x i16> %ins8,  i16 %a9,  i32 9
+  %ins10 = insertelement <16 x i16> %ins9,  i16 %a10, i32 10
+  %ins11 = insertelement <16 x i16> %ins10, i16 %a11, i32 11
+  %ins12 = insertelement <16 x i16> %ins11, i16 %a12, i32 12
+  %ins13 = insertelement <16 x i16> %ins12, i16 %a13, i32 13
+  %ins14 = insertelement <16 x i16> %ins13, i16 %a14, i32 14
+  %ins15 = insertelement <16 x i16> %ins14, i16 %a15, i32 15
+  ret <16 x i16> %ins15
+}
+
+define <32 x i8> @test_buildvector_v32i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15, i8 %a16, i8 %a17, i8 %a18, i8 %a19, i8 %a20, i8 %a21, i8 %a22, i8 %a23, i8 %a24, i8 %a25, i8 %a26, i8 %a27, i8 %a28, i8 %a29, i8 %a30, i8 %a31) {
+; AVX1-32-LABEL: test_buildvector_v32i8:
+; AVX1-32:       # BB#0:
+; AVX1-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX1-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-32-NEXT:    retl
+;
+; AVX1-64-LABEL: test_buildvector_v32i8:
+; AVX1-64:       # BB#0:
+; AVX1-64-NEXT:    vmovd %edi, %xmm0
+; AVX1-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX1-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-64-NEXT:    retq
+;
+; AVX2-32-LABEL: test_buildvector_v32i8:
+; AVX2-32:       # BB#0:
+; AVX2-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX2-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX2-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-32-NEXT:    retl
+;
+; AVX2-64-LABEL: test_buildvector_v32i8:
+; AVX2-64:       # BB#0:
+; AVX2-64-NEXT:    vmovd %edi, %xmm0
+; AVX2-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX2-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-64-NEXT:    retq
+  %ins0  = insertelement <32 x i8> undef,  i8 %a0,  i32 0
+  %ins1  = insertelement <32 x i8> %ins0,  i8 %a1,  i32 1
+  %ins2  = insertelement <32 x i8> %ins1,  i8 %a2,  i32 2
+  %ins3  = insertelement <32 x i8> %ins2,  i8 %a3,  i32 3
+  %ins4  = insertelement <32 x i8> %ins3,  i8 %a4,  i32 4
+  %ins5  = insertelement <32 x i8> %ins4,  i8 %a5,  i32 5
+  %ins6  = insertelement <32 x i8> %ins5,  i8 %a6,  i32 6
+  %ins7  = insertelement <32 x i8> %ins6,  i8 %a7,  i32 7
+  %ins8  = insertelement <32 x i8> %ins7,  i8 %a8,  i32 8
+  %ins9  = insertelement <32 x i8> %ins8,  i8 %a9,  i32 9
+  %ins10 = insertelement <32 x i8> %ins9,  i8 %a10, i32 10
+  %ins11 = insertelement <32 x i8> %ins10, i8 %a11, i32 11
+  %ins12 = insertelement <32 x i8> %ins11, i8 %a12, i32 12
+  %ins13 = insertelement <32 x i8> %ins12, i8 %a13, i32 13
+  %ins14 = insertelement <32 x i8> %ins13, i8 %a14, i32 14
+  %ins15 = insertelement <32 x i8> %ins14, i8 %a15, i32 15
+  %ins16 = insertelement <32 x i8> %ins15, i8 %a16, i32 16
+  %ins17 = insertelement <32 x i8> %ins16, i8 %a17, i32 17
+  %ins18 = insertelement <32 x i8> %ins17, i8 %a18, i32 18
+  %ins19 = insertelement <32 x i8> %ins18, i8 %a19, i32 19
+  %ins20 = insertelement <32 x i8> %ins19, i8 %a20, i32 20
+  %ins21 = insertelement <32 x i8> %ins20, i8 %a21, i32 21
+  %ins22 = insertelement <32 x i8> %ins21, i8 %a22, i32 22
+  %ins23 = insertelement <32 x i8> %ins22, i8 %a23, i32 23
+  %ins24 = insertelement <32 x i8> %ins23, i8 %a24, i32 24
+  %ins25 = insertelement <32 x i8> %ins24, i8 %a25, i32 25
+  %ins26 = insertelement <32 x i8> %ins25, i8 %a26, i32 26
+  %ins27 = insertelement <32 x i8> %ins26, i8 %a27, i32 27
+  %ins28 = insertelement <32 x i8> %ins27, i8 %a28, i32 28
+  %ins29 = insertelement <32 x i8> %ins28, i8 %a29, i32 29
+  %ins30 = insertelement <32 x i8> %ins29, i8 %a30, i32 30
+  %ins31 = insertelement <32 x i8> %ins30, i8 %a31, i32 31
+  ret <32 x i8> %ins31
+}
diff --git a/test/CodeGen/X86/build-vector-512.ll b/test/CodeGen/X86/build-vector-512.ll
new file mode 100644
index 000000000000..21737cca93a1
--- /dev/null
+++ b/test/CodeGen/X86/build-vector-512.ll
@@ -0,0 +1,712 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX-32 --check-prefix=AVX512F-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX-64 --check-prefix=AVX512F-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX-32 --check-prefix=AVX512BW-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX-64 --check-prefix=AVX512BW-64
+
+define <8 x double> @test_buildvector_v8f64(double %a0, double %a1, double %a2, double %a3, double %a4, double %a5, double %a6, double %a7) {
+; AVX-32-LABEL: test_buildvector_v8f64:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %zmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v8f64:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm6 = xmm6[0],xmm7[0]
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-64-NEXT:    vinsertf64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <8 x double> undef, double %a0, i32 0
+  %ins1 = insertelement <8 x double> %ins0, double %a1, i32 1
+  %ins2 = insertelement <8 x double> %ins1, double %a2, i32 2
+  %ins3 = insertelement <8 x double> %ins2, double %a3, i32 3
+  %ins4 = insertelement <8 x double> %ins3, double %a4, i32 4
+  %ins5 = insertelement <8 x double> %ins4, double %a5, i32 5
+  %ins6 = insertelement <8 x double> %ins5, double %a6, i32 6
+  %ins7 = insertelement <8 x double> %ins6, double %a7, i32 7
+  ret <8 x double> %ins7
+}
+
+define <16 x float> @test_buildvector_v16f32(float %a0, float %a1, float %a2, float %a3, float %a4, float %a5, float %a6, float %a7, float %a8, float %a9, float %a10, float %a11, float %a12, float %a13, float %a14, float %a15) {
+; AVX-32-LABEL: test_buildvector_v16f32:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %zmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v16f32:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX-64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
+; AVX-64-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
+; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX-64-NEXT:    retq
+  %ins0  = insertelement <16 x float> undef,  float %a0,  i32 0
+  %ins1  = insertelement <16 x float> %ins0,  float %a1,  i32 1
+  %ins2  = insertelement <16 x float> %ins1,  float %a2,  i32 2
+  %ins3  = insertelement <16 x float> %ins2,  float %a3,  i32 3
+  %ins4  = insertelement <16 x float> %ins3,  float %a4,  i32 4
+  %ins5  = insertelement <16 x float> %ins4,  float %a5,  i32 5
+  %ins6  = insertelement <16 x float> %ins5,  float %a6,  i32 6
+  %ins7  = insertelement <16 x float> %ins6,  float %a7,  i32 7
+  %ins8  = insertelement <16 x float> %ins7,  float %a8,  i32 8
+  %ins9  = insertelement <16 x float> %ins8,  float %a9,  i32 9
+  %ins10 = insertelement <16 x float> %ins9,  float %a10, i32 10
+  %ins11 = insertelement <16 x float> %ins10, float %a11, i32 11
+  %ins12 = insertelement <16 x float> %ins11, float %a12, i32 12
+  %ins13 = insertelement <16 x float> %ins12, float %a13, i32 13
+  %ins14 = insertelement <16 x float> %ins13, float %a14, i32 14
+  %ins15 = insertelement <16 x float> %ins14, float %a15, i32 15
+  ret <16 x float> %ins15
+}
+
+define <8 x i64> @test_buildvector_v8i64(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7) {
+; AVX-32-LABEL: test_buildvector_v8i64:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v8i64:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vmovq %rcx, %xmm0
+; AVX-64-NEXT:    vmovq %rdx, %xmm1
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-64-NEXT:    vmovq %rsi, %xmm1
+; AVX-64-NEXT:    vmovq %rdi, %xmm2
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-64-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX-64-NEXT:    vmovq %r9, %xmm1
+; AVX-64-NEXT:    vmovq %r8, %xmm2
+; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX-64-NEXT:    vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
+; AVX-64-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX-64-NEXT:    retq
+  %ins0 = insertelement <8 x i64> undef, i64 %a0, i32 0
+  %ins1 = insertelement <8 x i64> %ins0, i64 %a1, i32 1
+  %ins2 = insertelement <8 x i64> %ins1, i64 %a2, i32 2
+  %ins3 = insertelement <8 x i64> %ins2, i64 %a3, i32 3
+  %ins4 = insertelement <8 x i64> %ins3, i64 %a4, i32 4
+  %ins5 = insertelement <8 x i64> %ins4, i64 %a5, i32 5
+  %ins6 = insertelement <8 x i64> %ins5, i64 %a6, i32 6
+  %ins7 = insertelement <8 x i64> %ins6, i64 %a7, i32 7
+  ret <8 x i64> %ins7
+}
+
+define <16 x i32> @test_buildvector_v16i32(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i32 %a9, i32 %a10, i32 %a11, i32 %a12, i32 %a13, i32 %a14, i32 %a15) {
+; AVX-32-LABEL: test_buildvector_v16i32:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %zmm0
+; AVX-32-NEXT:    retl
+;
+; AVX-64-LABEL: test_buildvector_v16i32:
+; AVX-64:       # BB#0:
+; AVX-64-NEXT:    vmovd %edi, %xmm0
+; AVX-64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX-64-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX-64-NEXT:    vmovd %r8d, %xmm1
+; AVX-64-NEXT:    vpinsrd $1, %r9d, %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX-64-NEXT:    vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX-64-NEXT:    retq
+  %ins0  = insertelement <16 x i32> undef,  i32 %a0,  i32 0
+  %ins1  = insertelement <16 x i32> %ins0,  i32 %a1,  i32 1
+  %ins2  = insertelement <16 x i32> %ins1,  i32 %a2,  i32 2
+  %ins3  = insertelement <16 x i32> %ins2,  i32 %a3,  i32 3
+  %ins4  = insertelement <16 x i32> %ins3,  i32 %a4,  i32 4
+  %ins5  = insertelement <16 x i32> %ins4,  i32 %a5,  i32 5
+  %ins6  = insertelement <16 x i32> %ins5,  i32 %a6,  i32 6
+  %ins7  = insertelement <16 x i32> %ins6,  i32 %a7,  i32 7
+  %ins8  = insertelement <16 x i32> %ins7,  i32 %a8,  i32 8
+  %ins9  = insertelement <16 x i32> %ins8,  i32 %a9,  i32 9
+  %ins10 = insertelement <16 x i32> %ins9,  i32 %a10, i32 10
+  %ins11 = insertelement <16 x i32> %ins10, i32 %a11, i32 11
+  %ins12 = insertelement <16 x i32> %ins11, i32 %a12, i32 12
+  %ins13 = insertelement <16 x i32> %ins12, i32 %a13, i32 13
+  %ins14 = insertelement <16 x i32> %ins13, i32 %a14, i32 14
+  %ins15 = insertelement <16 x i32> %ins14, i32 %a15, i32 15
+  ret <16 x i32> %ins15
+}
+
+define <32 x i16> @test_buildvector_v32i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7, i16 %a8, i16 %a9, i16 %a10, i16 %a11, i16 %a12, i16 %a13, i16 %a14, i16 %a15, i16 %a16, i16 %a17, i16 %a18, i16 %a19, i16 %a20, i16 %a21, i16 %a22, i16 %a23, i16 %a24, i16 %a25, i16 %a26, i16 %a27, i16 %a28, i16 %a29, i16 %a30, i16 %a31) {
+; AVX512F-32-LABEL: test_buildvector_v32i16:
+; AVX512F-32:       # BB#0:
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: test_buildvector_v32i16:
+; AVX512F-64:       # BB#0:
+; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-64-NEXT:    vmovd %edi, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $1, %esi, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $2, %edx, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $3, %ecx, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $4, %r8d, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $5, %r9d, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512F-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512BW-32-LABEL: test_buildvector_v32i16:
+; AVX512BW-32:       # BB#0:
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrw $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrw $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrw $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrw $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrw $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrw $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-32-NEXT:    retl
+;
+; AVX512BW-64-LABEL: test_buildvector_v32i16:
+; AVX512BW-64:       # BB#0:
+; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512BW-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512BW-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-64-NEXT:    vmovd %edi, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $2, %edx, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $4, %r8d, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $5, %r9d, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512BW-64-NEXT:    vpinsrw $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrw $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrw $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrw $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrw $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrw $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrw $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-64-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-64-NEXT:    retq
+  %ins0  = insertelement <32 x i16> undef,  i16 %a0,  i32 0
+  %ins1  = insertelement <32 x i16> %ins0,  i16 %a1,  i32 1
+  %ins2  = insertelement <32 x i16> %ins1,  i16 %a2,  i32 2
+  %ins3  = insertelement <32 x i16> %ins2,  i16 %a3,  i32 3
+  %ins4  = insertelement <32 x i16> %ins3,  i16 %a4,  i32 4
+  %ins5  = insertelement <32 x i16> %ins4,  i16 %a5,  i32 5
+  %ins6  = insertelement <32 x i16> %ins5,  i16 %a6,  i32 6
+  %ins7  = insertelement <32 x i16> %ins6,  i16 %a7,  i32 7
+  %ins8  = insertelement <32 x i16> %ins7,  i16 %a8,  i32 8
+  %ins9  = insertelement <32 x i16> %ins8,  i16 %a9,  i32 9
+  %ins10 = insertelement <32 x i16> %ins9,  i16 %a10, i32 10
+  %ins11 = insertelement <32 x i16> %ins10, i16 %a11, i32 11
+  %ins12 = insertelement <32 x i16> %ins11, i16 %a12, i32 12
+  %ins13 = insertelement <32 x i16> %ins12, i16 %a13, i32 13
+  %ins14 = insertelement <32 x i16> %ins13, i16 %a14, i32 14
+  %ins15 = insertelement <32 x i16> %ins14, i16 %a15, i32 15
+  %ins16 = insertelement <32 x i16> %ins15, i16 %a16, i32 16
+  %ins17 = insertelement <32 x i16> %ins16, i16 %a17, i32 17
+  %ins18 = insertelement <32 x i16> %ins17, i16 %a18, i32 18
+  %ins19 = insertelement <32 x i16> %ins18, i16 %a19, i32 19
+  %ins20 = insertelement <32 x i16> %ins19, i16 %a20, i32 20
+  %ins21 = insertelement <32 x i16> %ins20, i16 %a21, i32 21
+  %ins22 = insertelement <32 x i16> %ins21, i16 %a22, i32 22
+  %ins23 = insertelement <32 x i16> %ins22, i16 %a23, i32 23
+  %ins24 = insertelement <32 x i16> %ins23, i16 %a24, i32 24
+  %ins25 = insertelement <32 x i16> %ins24, i16 %a25, i32 25
+  %ins26 = insertelement <32 x i16> %ins25, i16 %a26, i32 26
+  %ins27 = insertelement <32 x i16> %ins26, i16 %a27, i32 27
+  %ins28 = insertelement <32 x i16> %ins27, i16 %a28, i32 28
+  %ins29 = insertelement <32 x i16> %ins28, i16 %a29, i32 29
+  %ins30 = insertelement <32 x i16> %ins29, i16 %a30, i32 30
+  %ins31 = insertelement <32 x i16> %ins30, i16 %a31, i32 31
+  ret <32 x i16> %ins31
+}
+
+define <64 x i8> @test_buildvector_v64i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15, i8 %a16, i8 %a17, i8 %a18, i8 %a19, i8 %a20, i8 %a21, i8 %a22, i8 %a23, i8 %a24, i8 %a25, i8 %a26, i8 %a27, i8 %a28, i8 %a29, i8 %a30, i8 %a31, i8 %a32, i8 %a33, i8 %a34, i8 %a35, i8 %a36, i8 %a37, i8 %a38, i8 %a39, i8 %a40, i8 %a41, i8 %a42, i8 %a43, i8 %a44, i8 %a45, i8 %a46, i8 %a47, i8 %a48, i8 %a49, i8 %a50, i8 %a51, i8 %a52, i8 %a53, i8 %a54, i8 %a55, i8 %a56, i8 %a57, i8 %a58, i8 %a59, i8 %a60, i8 %a61, i8 %a62, i8 %a63) {
+; AVX512F-32-LABEL: test_buildvector_v64i8:
+; AVX512F-32:       # BB#0:
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512F-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512F-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512F-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-32-NEXT:    retl
+;
+; AVX512F-64-LABEL: test_buildvector_v64i8:
+; AVX512F-64:       # BB#0:
+; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-64-NEXT:    vmovd %edi, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512F-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512F-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512F-64-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512F-64-NEXT:    retq
+;
+; AVX512BW-32-LABEL: test_buildvector_v64i8:
+; AVX512BW-32:       # BB#0:
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX512BW-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512BW-32-NEXT:    vpinsrb $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $8, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $9, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $10, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $11, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $12, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $13, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $14, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vpinsrb $15, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX512BW-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-32-NEXT:    retl
+;
+; AVX512BW-64-LABEL: test_buildvector_v64i8:
+; AVX512BW-64:       # BB#0:
+; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512BW-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512BW-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-64-NEXT:    vmovd %edi, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $1, %esi, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $4, %r8d, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $5, %r9d, %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512BW-64-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512BW-64-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512BW-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-64-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-64-NEXT:    retq
+  %ins0  = insertelement <64 x i8> undef,  i8 %a0,  i32 0
+  %ins1  = insertelement <64 x i8> %ins0,  i8 %a1,  i32 1
+  %ins2  = insertelement <64 x i8> %ins1,  i8 %a2,  i32 2
+  %ins3  = insertelement <64 x i8> %ins2,  i8 %a3,  i32 3
+  %ins4  = insertelement <64 x i8> %ins3,  i8 %a4,  i32 4
+  %ins5  = insertelement <64 x i8> %ins4,  i8 %a5,  i32 5
+  %ins6  = insertelement <64 x i8> %ins5,  i8 %a6,  i32 6
+  %ins7  = insertelement <64 x i8> %ins6,  i8 %a7,  i32 7
+  %ins8  = insertelement <64 x i8> %ins7,  i8 %a8,  i32 8
+  %ins9  = insertelement <64 x i8> %ins8,  i8 %a9,  i32 9
+  %ins10 = insertelement <64 x i8> %ins9,  i8 %a10, i32 10
+  %ins11 = insertelement <64 x i8> %ins10, i8 %a11, i32 11
+  %ins12 = insertelement <64 x i8> %ins11, i8 %a12, i32 12
+  %ins13 = insertelement <64 x i8> %ins12, i8 %a13, i32 13
+  %ins14 = insertelement <64 x i8> %ins13, i8 %a14, i32 14
+  %ins15 = insertelement <64 x i8> %ins14, i8 %a15, i32 15
+  %ins16 = insertelement <64 x i8> %ins15, i8 %a16, i32 16
+  %ins17 = insertelement <64 x i8> %ins16, i8 %a17, i32 17
+  %ins18 = insertelement <64 x i8> %ins17, i8 %a18, i32 18
+  %ins19 = insertelement <64 x i8> %ins18, i8 %a19, i32 19
+  %ins20 = insertelement <64 x i8> %ins19, i8 %a20, i32 20
+  %ins21 = insertelement <64 x i8> %ins20, i8 %a21, i32 21
+  %ins22 = insertelement <64 x i8> %ins21, i8 %a22, i32 22
+  %ins23 = insertelement <64 x i8> %ins22, i8 %a23, i32 23
+  %ins24 = insertelement <64 x i8> %ins23, i8 %a24, i32 24
+  %ins25 = insertelement <64 x i8> %ins24, i8 %a25, i32 25
+  %ins26 = insertelement <64 x i8> %ins25, i8 %a26, i32 26
+  %ins27 = insertelement <64 x i8> %ins26, i8 %a27, i32 27
+  %ins28 = insertelement <64 x i8> %ins27, i8 %a28, i32 28
+  %ins29 = insertelement <64 x i8> %ins28, i8 %a29, i32 29
+  %ins30 = insertelement <64 x i8> %ins29, i8 %a30, i32 30
+  %ins31 = insertelement <64 x i8> %ins30, i8 %a31, i32 31
+  %ins32 = insertelement <64 x i8> %ins31, i8 %a32, i32 32
+  %ins33 = insertelement <64 x i8> %ins32, i8 %a33, i32 33
+  %ins34 = insertelement <64 x i8> %ins33, i8 %a34, i32 34
+  %ins35 = insertelement <64 x i8> %ins34, i8 %a35, i32 35
+  %ins36 = insertelement <64 x i8> %ins35, i8 %a36, i32 36
+  %ins37 = insertelement <64 x i8> %ins36, i8 %a37, i32 37
+  %ins38 = insertelement <64 x i8> %ins37, i8 %a38, i32 38
+  %ins39 = insertelement <64 x i8> %ins38, i8 %a39, i32 39
+  %ins40 = insertelement <64 x i8> %ins39, i8 %a40, i32 40
+  %ins41 = insertelement <64 x i8> %ins40, i8 %a41, i32 41
+  %ins42 = insertelement <64 x i8> %ins41, i8 %a42, i32 42
+  %ins43 = insertelement <64 x i8> %ins42, i8 %a43, i32 43
+  %ins44 = insertelement <64 x i8> %ins43, i8 %a44, i32 44
+  %ins45 = insertelement <64 x i8> %ins44, i8 %a45, i32 45
+  %ins46 = insertelement <64 x i8> %ins45, i8 %a46, i32 46
+  %ins47 = insertelement <64 x i8> %ins46, i8 %a47, i32 47
+  %ins48 = insertelement <64 x i8> %ins47, i8 %a48, i32 48
+  %ins49 = insertelement <64 x i8> %ins48, i8 %a49, i32 49
+  %ins50 = insertelement <64 x i8> %ins49, i8 %a50, i32 50
+  %ins51 = insertelement <64 x i8> %ins50, i8 %a51, i32 51
+  %ins52 = insertelement <64 x i8> %ins51, i8 %a52, i32 52
+  %ins53 = insertelement <64 x i8> %ins52, i8 %a53, i32 53
+  %ins54 = insertelement <64 x i8> %ins53, i8 %a54, i32 54
+  %ins55 = insertelement <64 x i8> %ins54, i8 %a55, i32 55
+  %ins56 = insertelement <64 x i8> %ins55, i8 %a56, i32 56
+  %ins57 = insertelement <64 x i8> %ins56, i8 %a57, i32 57
+  %ins58 = insertelement <64 x i8> %ins57, i8 %a58, i32 58
+  %ins59 = insertelement <64 x i8> %ins58, i8 %a59, i32 59
+  %ins60 = insertelement <64 x i8> %ins59, i8 %a60, i32 60
+  %ins61 = insertelement <64 x i8> %ins60, i8 %a61, i32 61
+  %ins62 = insertelement <64 x i8> %ins61, i8 %a62, i32 62
+  %ins63 = insertelement <64 x i8> %ins62, i8 %a63, i32 63
+  ret <64 x i8> %ins63
+}
diff --git a/test/CodeGen/X86/combine-abs.ll b/test/CodeGen/X86/combine-abs.ll
index ac8f790a2ead..887abe99f6ed 100644
--- a/test/CodeGen/X86/combine-abs.ll
+++ b/test/CodeGen/X86/combine-abs.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s  --check-prefix=CHECK  --check-prefix=AVX512  --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s  --check-prefix=CHECK  --check-prefix=AVX512  --check-prefix=AVX512VL
 
 ; fold (abs c1) -> c2
 define <4 x i32> @combine_v4i32_abs_constant() {
@@ -27,10 +29,10 @@ define <8 x i16> @combine_v8i16_abs_abs(<8 x i16> %a) {
 ; CHECK-NEXT:    vpabsw %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %a1 = call <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16> %a)
-  %n2 = sub <8 x i16> zeroinitializer, %a1
-  %c2 = icmp slt <8 x i16> %a1, zeroinitializer
-  %a2 = select <8 x i1> %c2, <8 x i16> %n2, <8 x i16> %a1
-  ret <8 x i16> %a2
+  %s2 = ashr <8 x i16> %a1, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %a2 = add <8 x i16> %a1, %s2
+  %x2 = xor <8 x i16> %a2, %s2
+  ret <8 x i16> %x2
 }
 
 define <32 x i8> @combine_v32i8_abs_abs(<32 x i8> %a) {
@@ -46,17 +48,29 @@ define <32 x i8> @combine_v32i8_abs_abs(<32 x i8> %a) {
 }
 
 define <4 x i64> @combine_v4i64_abs_abs(<4 x i64> %a) {
-; CHECK-LABEL: combine_v4i64_abs_abs:
-; CHECK:       # BB#0:
-; CHECK-NEXT:    vpsrad $31, %ymm0, %ymm1
-; CHECK-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,1,3,3,5,5,7,7]
-; CHECK-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    vpsrad $31, %ymm0, %ymm1
-; CHECK-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,1,3,3,5,5,7,7]
-; CHECK-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; AVX2-LABEL: combine_v4i64_abs_abs:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm1
+; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,1,3,3,5,5,7,7]
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm1
+; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,1,3,3,5,5,7,7]
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: combine_v4i64_abs_abs:
+; AVX512F:       # BB#0:
+; AVX512F-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
+; AVX512F-NEXT:    vpabsq %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: combine_v4i64_abs_abs:
+; AVX512VL:       # BB#0:
+; AVX512VL-NEXT:    vpabsq %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
   %n1 = sub <4 x i64> zeroinitializer, %a
   %b1 = icmp slt <4 x i64> %a, zeroinitializer
   %a1 = select <4 x i1> %b1, <4 x i64> %n1, <4 x i64> %a
diff --git a/test/CodeGen/X86/commuted-blend-mask.ll b/test/CodeGen/X86/commuted-blend-mask.ll
index e6322cbb7a14..37830509d5a2 100644
--- a/test/CodeGen/X86/commuted-blend-mask.ll
+++ b/test/CodeGen/X86/commuted-blend-mask.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s
 
 ; When commuting the operands of a SSE blend, make sure that the resulting blend
 ; mask can be encoded as a imm8.
@@ -7,7 +8,7 @@
 ;   pblendw $4294967103, %xmm1, %xmm0
 
 define <4 x i32> @test(<4 x i32> %a, <4 x i32> %b) {
-  ;CHECK: pblendw $63, %xmm1, %xmm0
+; CHECK: pblendw $63, %xmm1, %xmm0
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
   ret <4 x i32> %shuffle
 }
diff --git a/test/CodeGen/X86/ctpop-combine.ll b/test/CodeGen/X86/ctpop-combine.ll
index b7031a817e82..bbfc2ead04c6 100644
--- a/test/CodeGen/X86/ctpop-combine.ll
+++ b/test/CodeGen/X86/ctpop-combine.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=corei7 | FileCheck %s
 
+declare i8 @llvm.ctpop.i8(i8) nounwind readnone
 declare i64 @llvm.ctpop.i64(i64) nounwind readnone
 
 define i32 @test1(i64 %x) nounwind readnone {
@@ -48,3 +49,16 @@ define i32 @test3(i64 %x) nounwind readnone {
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
+
+define i8 @test4(i8 %x) nounwind readnone {
+; CHECK-LABEL: test4:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    andl $127, %edi
+; CHECK-NEXT:    popcntw %di, %ax
+; CHECK-NEXT:    # kill: %AL<def> %AL<kill> %AX<kill>
+; CHECK-NEXT:    retq
+  %x2 = and i8 %x, 127
+  %count = tail call i8 @llvm.ctpop.i8(i8 %x2)
+  %and = and i8 %count, 7
+  ret i8 %and
+}
diff --git a/test/CodeGen/X86/dbg-baseptr.ll b/test/CodeGen/X86/dbg-baseptr.ll
index f69c78af7367..fb0da1b50d11 100644
--- a/test/CodeGen/X86/dbg-baseptr.ll
+++ b/test/CodeGen/X86/dbg-baseptr.ll
@@ -16,12 +16,12 @@ define i32 @f0(%struct.s* byval align 8 %input) !dbg !8 {
 
 ; CHECK-LABEL: f1:
 ; CHECK: DEBUG_VALUE: f:input <- [%RBP+16]
-define i32 @f1(%struct.s* byval align 8 %input) !dbg !8 {
+define i32 @f1(%struct.s* byval align 8 %input) !dbg !19 {
   %val = load i64, i64* @glob
   ; this alloca should force FP usage.
   %stackspace = alloca i32, i64 %val, align 1
   store i32* %stackspace, i32** @ptr
-  call void @llvm.dbg.declare(metadata %struct.s* %input, metadata !4, metadata !17), !dbg !18
+  call void @llvm.dbg.declare(metadata %struct.s* %input, metadata !20, metadata !17), !dbg !21
   ret i32 42
 }
 
@@ -37,11 +37,11 @@ define i32 @f1(%struct.s* byval align 8 %input) !dbg !8 {
 ; The parameter should still be referenced through RBP though.
 ; CHECK-NOT: DEBUG_VALUE: f:input <- [%RBX
 ; CHECK: DEBUG_VALUE: f:input <- [%RBP+16]
-define i32 @f2(%struct.s* byval align 8 %input) !dbg !8 {
+define i32 @f2(%struct.s* byval align 8 %input) !dbg !22 {
   %val = load i64, i64* @glob
   %stackspace = alloca i32, i64 %val, align 64
   store i32* %stackspace, i32** @ptr
-  call void @llvm.dbg.declare(metadata %struct.s* %input, metadata !4, metadata !17), !dbg !18
+  call void @llvm.dbg.declare(metadata %struct.s* %input, metadata !23, metadata !17), !dbg !24
   ret i32 42
 }
 
@@ -73,3 +73,10 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 !17 = !DIExpression()
 !18 = !DILocation(line: 5, scope: !8)
+
+!19 = distinct !DISubprogram(name: "f", file: !3, line: 5, type: !6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, unit: !2, variables: !5)
+!20 = !DILocalVariable(name: "input", arg: 1, scope: !19, file: !3, line: 5, type: !9)
+!21 = !DILocation(line: 5, scope: !19)
+!22 = distinct !DISubprogram(name: "f", file: !3, line: 5, type: !6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, unit: !2, variables: !5)
+!23 = !DILocalVariable(name: "input", arg: 1, scope: !22, file: !3, line: 5, type: !9)
+!24 = !DILocation(line: 5, scope: !22)
diff --git a/test/CodeGen/X86/eflags-copy-expansion.mir b/test/CodeGen/X86/eflags-copy-expansion.mir
index 36044b4d2059..28f47c3c2496 100644
--- a/test/CodeGen/X86/eflags-copy-expansion.mir
+++ b/test/CodeGen/X86/eflags-copy-expansion.mir
@@ -25,7 +25,6 @@ liveins:
 body:             |
   bb.0.entry:
     liveins: %edi
-    successors: %bb.1.false
     NOOP implicit-def %al
 
     ; The bug was triggered only when LivePhysReg is used, which
diff --git a/test/CodeGen/X86/frame-lowering-debug-intrinsic.ll b/test/CodeGen/X86/frame-lowering-debug-intrinsic.ll
index e86d094ac341..f9ecf707810b 100644
--- a/test/CodeGen/X86/frame-lowering-debug-intrinsic.ll
+++ b/test/CodeGen/X86/frame-lowering-debug-intrinsic.ll
@@ -35,8 +35,8 @@ define void @fn2NoDebug(%struct.Buffer* byval align 64 %p1) {
 ; CHECK-NEXT: pop
 ; CHECK-NEXT: ret
 
-define void @fn2WithDebug(%struct.Buffer* byval align 64 %p1) !dbg !4 {
-  call void @llvm.dbg.declare(metadata %struct.Buffer* %p1, metadata !5, metadata !6), !dbg !7
+define void @fn2WithDebug(%struct.Buffer* byval align 64 %p1) !dbg !8 {
+  call void @llvm.dbg.declare(metadata %struct.Buffer* %p1, metadata !9, metadata !6), !dbg !10
   ret void
 }
 
@@ -64,3 +64,6 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 !5 = !DILocalVariable(name: "w", scope: !4)
 !6 = !DIExpression()
 !7 = !DILocation(line: 210, column: 12, scope: !4)
+!8 = distinct !DISubprogram(name: "withDebug", unit: !0)
+!9 = !DILocalVariable(name: "w", scope: !8)
+!10 = !DILocation(line: 210, column: 12, scope: !8)
diff --git a/test/CodeGen/X86/implicit-null-checks.mir b/test/CodeGen/X86/implicit-null-checks.mir
index 39bfedaa7814..d0ba057fa009 100644
--- a/test/CodeGen/X86/implicit-null-checks.mir
+++ b/test/CodeGen/X86/implicit-null-checks.mir
@@ -384,14 +384,12 @@ liveins:
 
 body:             |
   bb.0.entry:
-    successors: %bb.3.is_null, %bb.1.not_null
     liveins: %esi, %rdi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
     JE_1 %bb.3.is_null, implicit %eflags
 
   bb.1.not_null:
-    successors: %bb.4.ret_100, %bb.2.ret_200
     liveins: %esi, %rdi
 
     %eax = MOV32ri 2200000
@@ -427,7 +425,6 @@ liveins:
 
 body:             |
   bb.0.entry:
-    successors: %bb.3.is_null, %bb.1.not_null
     liveins: %esi, %rdi, %rdx
 
     %eax = MOV32rm killed %rdx, 1, _, 0, _ :: (volatile load 4 from %ir.ptr)
@@ -435,7 +432,6 @@ body:             |
     JE_1 %bb.3.is_null, implicit %eflags
 
   bb.1.not_null:
-    successors: %bb.4.ret_100, %bb.2.ret_200
     liveins: %esi, %rdi
 
     %eax = MOV32ri 2200000
@@ -444,7 +440,6 @@ body:             |
     JE_1 %bb.4.ret_100, implicit %eflags
 
   bb.2.ret_200:
-    successors: %bb.3.is_null
 
     %eax = MOV32ri 200
 
@@ -472,14 +467,12 @@ liveins:
 
 body:             |
   bb.0.entry:
-    successors: %bb.3.is_null, %bb.1.not_null
     liveins: %esi, %rdi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
     JE_1 %bb.3.is_null, implicit %eflags
 
   bb.1.not_null:
-    successors: %bb.4.ret_100, %bb.2.ret_200
     liveins: %esi, %rdi
 
     %eax = MOV32ri 2200000
@@ -515,14 +508,12 @@ liveins:
 
 body:             |
   bb.0.entry:
-    successors: %bb.3.is_null, %bb.1.not_null
     liveins: %rsi, %rdi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
     JE_1 %bb.3.is_null, implicit %eflags
 
   bb.1.not_null:
-    successors: %bb.4.ret_100, %bb.2.ret_200
     liveins: %rsi, %rdi
 
     %rdi  = MOV64ri 5000
@@ -557,14 +548,12 @@ liveins:
 
 body:             |
   bb.0.entry:
-    successors: %bb.3.is_null, %bb.1.not_null
     liveins: %rsi, %rdi, %rdx
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
     JE_1 %bb.3.is_null, implicit %eflags
 
   bb.1.not_null:
-    successors: %bb.4.ret_100, %bb.2.ret_200
     liveins: %rsi, %rdi, %rdx
 
     %rbx  = MOV64rr %rdx
@@ -603,7 +592,6 @@ calleeSavedRegisters: [ '%bh', '%bl', '%bp', '%bpl', '%bx', '%ebp', '%ebx',
 # CHECK: CALL64pcrel32
 body:             |
   bb.0.entry:
-    successors: %bb.2.leave, %bb.1.stay
     liveins: %rdi, %rbx
 
     frame-setup PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp
@@ -645,7 +633,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -680,7 +667,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -712,7 +698,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.1.is_null(0x30000000), %bb.2.not_null(0x50000000)
     liveins: %rsi, %rdi
   
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -745,7 +730,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.1.is_null(0x30000000), %bb.2.not_null(0x50000000)
     liveins: %rsi, %rdi
   
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -779,7 +763,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -810,7 +793,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -842,7 +824,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -874,7 +855,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -910,7 +890,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -941,7 +920,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -974,7 +952,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1006,7 +983,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1042,7 +1018,6 @@ calleeSavedRegisters: [ '%bh', '%bl', '%bp', '%bpl', '%bx', '%ebp', '%ebx',
                         '%r14d', '%r15d', '%r12w', '%r13w', '%r14w', '%r15w' ]
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rbx
 
     frame-setup PUSH64r killed %rbx, implicit-def %rsp, implicit %rsp
@@ -1082,7 +1057,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1116,7 +1090,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1149,7 +1122,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1182,7 +1154,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1214,7 +1185,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1246,7 +1216,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
@@ -1279,7 +1248,6 @@ liveins:
   - { reg: '%rsi' }
 body:             |
   bb.0.entry:
-    successors: %bb.2.is_null, %bb.1.not_null
     liveins: %rdi, %rsi
 
     TEST64rr %rdi, %rdi, implicit-def %eflags
diff --git a/test/CodeGen/X86/invalid-liveness.mir b/test/CodeGen/X86/invalid-liveness.mir
index ca862472ba86..c1da65e0be69 100644
--- a/test/CodeGen/X86/invalid-liveness.mir
+++ b/test/CodeGen/X86/invalid-liveness.mir
@@ -16,12 +16,10 @@ registers:
   - { id: 0, class: gr32 }
 body: |
   bb.0:
-    successors: %bb.2, %bb.3
     JG_1 %bb.2, implicit %eflags
     JMP_1 %bb.3
 
   bb.2:
-    successors: %bb.3
     %0 = IMPLICIT_DEF
     JMP_1 %bb.3
 
diff --git a/test/CodeGen/X86/machine-region-info.mir b/test/CodeGen/X86/machine-region-info.mir
index 0998fe97c235..78823a3eb006 100644
--- a/test/CodeGen/X86/machine-region-info.mir
+++ b/test/CodeGen/X86/machine-region-info.mir
@@ -4,67 +4,48 @@
 name:            fun
 body:             |
   bb.0:
-    successors: %bb.1, %bb.7
-
     CMP32ri8 %edi, 40, implicit-def %eflags
     JNE_1 %bb.7, implicit killed %eflags
     JMP_1 %bb.1
 
   bb.1:
-    successors: %bb.2, %bb.11
-
     CMP32ri8 %edi, 1, implicit-def %eflags
     JNE_1 %bb.11, implicit killed %eflags
     JMP_1 %bb.2
 
   bb.2:
-    successors: %bb.3, %bb.5
-
     CMP32ri8 %edi, 2, implicit-def %eflags
     JNE_1 %bb.5, implicit killed %eflags
     JMP_1 %bb.3
 
   bb.3:
-    successors: %bb.4, %bb.5
-
     CMP32ri8 %edi, 90, implicit-def %eflags
     JNE_1 %bb.5, implicit killed %eflags
     JMP_1 %bb.4
 
   bb.4:
-    successors: %bb.5
 
   bb.5:
-    successors: %bb.6, %bb.11
-
     CMP32ri8 %edi, 4, implicit-def %eflags
     JNE_1 %bb.11, implicit killed %eflags
     JMP_1 %bb.6
 
   bb.6:
-    successors: %bb.11
-
     JMP_1 %bb.11
 
   bb.7:
-    successors: %bb.9, %bb.8
-
     CMP32ri8 %edi, 5, implicit-def %eflags
     JE_1 %bb.9, implicit killed %eflags
     JMP_1 %bb.8
 
   bb.8:
-    successors: %bb.9
 
   bb.9:
-    successors: %bb.11, %bb.10
-
     CMP32ri8 %edi, 6, implicit-def %eflags
     JE_1 %bb.11, implicit killed %eflags
     JMP_1 %bb.10
 
   bb.10:
-    successors: %bb.11
 
   bb.11:
     RET 0
@@ -74,10 +55,10 @@ body:             |
 # CHECK: Region tree:
 # CHECK-NEXT: [0] BB#0 => <Function Return>
 # CHECK-NEXT:   [1] BB#0 => BB#11
+# CHECK-NEXT:     [2] BB#7 => BB#9
+# CHECK-NEXT:     [2] BB#9 => BB#11
 # CHECK-NEXT:     [2] BB#1 => BB#11
 # CHECK-NEXT:       [3] BB#2 => BB#5
 # CHECK-NEXT:         [4] BB#3 => BB#5
 # CHECK-NEXT:       [3] BB#5 => BB#11
-# CHECK-NEXT:     [2] BB#7 => BB#9
-# CHECK-NEXT:     [2] BB#9 => BB#11
 # CHECK-NEXT: End region tree
diff --git a/test/CodeGen/X86/ms-inline-asm-avx512.ll b/test/CodeGen/X86/ms-inline-asm-avx512.ll
new file mode 100644
index 000000000000..be60f5bca161
--- /dev/null
+++ b/test/CodeGen/X86/ms-inline-asm-avx512.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s | FileCheck %s
+
+; Generated from clang/test/CodeGen/ms-inline-asm-avx512.c
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; Function Attrs: noinline nounwind
+define void @ignore_fe_size() #0 {
+entry:
+  %c = alloca i8, align 1
+  call void asm sideeffect inteldialect "vaddps xmm1, xmm2, $1{1to4}\0A\09vaddps xmm1, xmm2, $2\0A\09mov eax, $3\0A\09mov $0, rax", "=*m,*m,*m,*m,~{eax},~{xmm1},~{dirflag},~{fpsr},~{flags}"(i8* %c, i8* %c, i8* %c, i8* %c) #1
+  ret void
+}
+
+; CHECK-LABEL: ignore_fe_size:
+; CHECK: vaddps  7(%rsp){1to4}, %xmm2, %xmm1
+; CHECK: vaddps  7(%rsp), %xmm2, %xmm1
+; CHECK: movl    7(%rsp), %eax
+; CHECK: movq    %rax, 7(%rsp)
+; CHECK: retq
+
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
diff --git a/test/CodeGen/X86/pr27681.mir b/test/CodeGen/X86/pr27681.mir
index 3e931b182e4e..002761bc1e68 100644
--- a/test/CodeGen/X86/pr27681.mir
+++ b/test/CodeGen/X86/pr27681.mir
@@ -25,7 +25,6 @@ stack:
   - { id: 2, type: spill-slot, offset: -32, size: 4, alignment: 4 }
 body:             |
   bb.0:
-    successors: %bb.1
     liveins: %ebp, %ebx, %edi, %esi
 
     frame-setup PUSH32r killed %ebp, implicit-def %esp, implicit %esp
@@ -41,7 +40,6 @@ body:             |
     %edx = MOV32ri 6
 
   bb.1:
-    successors: %bb.3, %bb.2
     liveins: %eax, %ebp, %ebx, %ecx, %edi, %edx
 
     %ebp = SHR32rCL killed %ebp, implicit-def dead %eflags, implicit %cl
@@ -66,7 +64,6 @@ body:             |
     JE_1 %bb.3, implicit %eflags
 
   bb.2:
-    successors: %bb.3
     liveins: %cl, %eax, %ebp, %esi
 
     OR32mr %esp, 1, _, 8, _, killed %eax, implicit-def %eflags ; :: (store 4 into %stack.1)
diff --git a/test/CodeGen/X86/pr32907.ll b/test/CodeGen/X86/pr32907.ll
new file mode 100644
index 000000000000..bc03fbe06843
--- /dev/null
+++ b/test/CodeGen/X86/pr32907.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512
+
+define <2 x i64> @PR32907(<2 x i64> %astype.i, <2 x i64> %astype6.i) {
+; SSE-LABEL: PR32907:
+; SSE:       # BB#0: # %entry
+; SSE-NEXT:    psubq %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    psubq %xmm0, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pandn %xmm0, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX2-LABEL: PR32907:
+; AVX2:       # BB#0: # %entry
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubq %xmm0, %xmm2, %xmm2
+; AVX2-NEXT:    vpandn %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: PR32907:
+; AVX512:       # BB#0: # %entry
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsraq $63, %zmm0, %zmm1
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpsubq %xmm0, %xmm2, %xmm2
+; AVX512-NEXT:    vpandn %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+entry:
+  %sub13.i = sub <2 x i64> %astype.i, %astype6.i
+  %x.lobit.i.i = ashr <2 x i64> %sub13.i, <i64 63, i64 63>
+  %sub.i.i = sub <2 x i64> zeroinitializer, %sub13.i
+  %0 = xor <2 x i64> %x.lobit.i.i, <i64 -1, i64 -1>
+  %1 = and <2 x i64> %sub13.i, %0
+  %2 = and <2 x i64> %x.lobit.i.i, %sub.i.i
+  %cond.i.i = or <2 x i64> %1, %2
+  ret <2 x i64> %cond.i.i
+}
diff --git a/test/CodeGen/X86/pre-coalesce.mir b/test/CodeGen/X86/pre-coalesce.mir
index 11805fe090b4..17d447dd097b 100644
--- a/test/CodeGen/X86/pre-coalesce.mir
+++ b/test/CodeGen/X86/pre-coalesce.mir
@@ -83,8 +83,6 @@ frameInfo:
   hasMustTailInVarArgFunc: false
 body:             |
   bb.0.entry:
-    successors: %bb.4(0x30000000), %bb.1.while.body.preheader(0x50000000)
-  
     %0 = MOV64rm %rip, 1, _, @b, _ :: (dereferenceable load 8 from @b)
     %12 = MOV8rm %0, 1, _, 0, _ :: (load 1 from %ir.t0)
     TEST8rr %12, %12, implicit-def %eflags
@@ -92,17 +90,12 @@ body:             |
     JNE_1 %bb.1.while.body.preheader, implicit killed %eflags
   
   bb.4:
-    successors: %bb.3.while.end(0x80000000)
-  
     %10 = COPY %11
     JMP_1 %bb.3.while.end
   
   bb.1.while.body.preheader:
-    successors: %bb.2.while.body(0x80000000)
 
   bb.2.while.body:
-    successors: %bb.3.while.end(0x04000000), %bb.2.while.body(0x7c000000)
-  
     %8 = MOVSX32rr8 %12
     %10 = COPY %11
     %10 = SHL32ri %10, 5, implicit-def dead %eflags
diff --git a/test/CodeGen/X86/regcall-no-plt.ll b/test/CodeGen/X86/regcall-no-plt.ll
new file mode 100644
index 000000000000..d525448b60ca
--- /dev/null
+++ b/test/CodeGen/X86/regcall-no-plt.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-freebsd -relocation-model=pic < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; According to x86-64 psABI, xmm0-xmm7 can be used to pass function parameters.  
+;; However regcall calling convention uses also xmm8-xmm15 to pass function  
+;; parameters which violates x86-64 psABI. 
+;; Detail info about it can be found at:
+;; https://sourceware.org/bugzilla/show_bug.cgi?id=21265
+;;
+;; We encounter the violation symptom when using PIC with lazy binding 
+;; optimization.
+;; In that case the PLT mechanism as described in x86_64 psABI will
+;; not preserve xmm8-xmm15 registers and will lead to miscompilation.
+;;
+;; The agreed solution is to disable PLT for regcall calling convention for 
+;; SystemV using ELF format.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+declare void @lazy()
+declare x86_regcallcc void @regcall_not_lazy()
+
+; CHECK-LABEL: foo:
+; CHECK:  callq lazy@PLT
+; CHECK:  callq *regcall_not_lazy@GOTPCREL(%rip)
+define void @foo() nounwind {
+  call void @lazy()
+  call void @regcall_not_lazy()
+  ret void
+}
+
+; CHECK-LABEL: tail_call_regcall:
+; CHECK:   jmpq *regcall_not_lazy@GOTPCREL(%rip)
+define void @tail_call_regcall() nounwind {
+  tail call void @regcall_not_lazy()
+  ret void
+}
+
+; CHECK-LABEL: tail_call_regular:
+; CHECK:   jmp lazy
+define void @tail_call_regular() nounwind {
+  tail call void @lazy()
+  ret void
+}
diff --git a/test/CodeGen/X86/shuffle-vs-trunc-512.ll b/test/CodeGen/X86/shuffle-vs-trunc-512.ll
index d053c63dcdb3..a3ba58975800 100644
--- a/test/CodeGen/X86/shuffle-vs-trunc-512.ll
+++ b/test/CodeGen/X86/shuffle-vs-trunc-512.ll
@@ -392,8 +392,10 @@ define void @shuffle_v32i16_to_v8i16(<32 x i16>* %L, <8 x i16>* %S) nounwind {
 ; AVX512BW-LABEL: shuffle_v32i16_to_v8i16:
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqu16 (%rdi), %zmm0
+; AVX512BW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX512BW-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vextracti32x4 $1, %zmm0, %xmm2
 ; AVX512BW-NEXT:    vmovd %xmm2, %eax
 ; AVX512BW-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
@@ -416,8 +418,10 @@ define void @shuffle_v32i16_to_v8i16(<32 x i16>* %L, <8 x i16>* %S) nounwind {
 ; AVX512BWVL-LABEL: shuffle_v32i16_to_v8i16:
 ; AVX512BWVL:       # BB#0:
 ; AVX512BWVL-NEXT:    vmovdqu16 (%rdi), %zmm0
+; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vmovss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; AVX512BWVL-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX512BWVL-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vextracti32x4 $1, %zmm0, %xmm2
 ; AVX512BWVL-NEXT:    vmovd %xmm2, %eax
 ; AVX512BWVL-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
diff --git a/test/CodeGen/X86/stack-folding-int-avx512.ll b/test/CodeGen/X86/stack-folding-int-avx512.ll
index 04a7d1159014..38e19efb7132 100644
--- a/test/CodeGen/X86/stack-folding-int-avx512.ll
+++ b/test/CodeGen/X86/stack-folding-int-avx512.ll
@@ -204,8 +204,8 @@ define <64 x i8> @stack_fold_pabsb_maskz(<64 x i8> %a0, i64 %mask) {
 }
 
 define <16 x i32> @stack_fold_pabsd(<16 x i32> %a0) {
-  ;check-label: stack_fold_pabsd
-  ;check:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte folded reload
+  ;CHECK-LABEL: stack_fold_pabsd
+  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %a0, <16 x i32> undef, i16 -1)
   ret <16 x i32> %2
@@ -213,16 +213,16 @@ define <16 x i32> @stack_fold_pabsd(<16 x i32> %a0) {
 declare <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32>, <16 x i32>, i16) nounwind readnone
 
 define <16 x i32> @stack_fold_pabsd_mask(<16 x i32> %passthru, <16 x i32> %a0, i16 %mask) {
-  ;check-label: stack_fold_pabsd
-  ;check:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte folded reload
+  ;CHECK-LABEL: stack_fold_pabsd_mask
+  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask)
   ret <16 x i32> %2
 }
 
 define <16 x i32> @stack_fold_pabsd_maskz(<16 x i32> %a0, i16 %mask) {
-  ;check-label: stack_fold_pabsd
-  ;check:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte folded reload
+  ;CHECK-LABEL: stack_fold_pabsd_maskz
+  ;CHECK:       vpabsd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %a0, <16 x i32> zeroinitializer, i16 %mask)
   ret <16 x i32> %2
diff --git a/test/CodeGen/X86/vec_partial.ll b/test/CodeGen/X86/vec_partial.ll
index e5ac81add7f6..ee15c2af6dd2 100644
--- a/test/CodeGen/X86/vec_partial.ll
+++ b/test/CodeGen/X86/vec_partial.ll
@@ -1,12 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64
 
 ; PR11580
 define <3 x float> @addf3(<3 x float> %x) {
-; CHECK-LABEL: addf3:
-; CHECK:       # BB#0: # %entry
-; CHECK-NEXT:    addps {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    retq
+; X86-LABEL: addf3:
+; X86:       # BB#0: # %entry
+; X86-NEXT:    addps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: addf3:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    addps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
 entry:
   %add = fadd <3 x float> %x, <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
   ret <3 x float> %add
@@ -14,9 +20,13 @@ entry:
 
 ; PR11580
 define <4 x float> @cvtf3_f4(<3 x float> %x) {
-; CHECK-LABEL: cvtf3_f4:
-; CHECK:       # BB#0: # %entry
-; CHECK-NEXT:    retq
+; X86-LABEL: cvtf3_f4:
+; X86:       # BB#0: # %entry
+; X86-NEXT:    retl
+;
+; X64-LABEL: cvtf3_f4:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    retq
 entry:
   %extractVec = shufflevector <3 x float> %x, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   ret <4 x float> %extractVec
@@ -24,9 +34,13 @@ entry:
 
 ; PR11580
 define <3 x float> @cvtf4_f3(<4 x float> %x) {
-; CHECK-LABEL: cvtf4_f3:
-; CHECK:       # BB#0: # %entry
-; CHECK-NEXT:    retq
+; X86-LABEL: cvtf4_f3:
+; X86:       # BB#0: # %entry
+; X86-NEXT:    retl
+;
+; X64-LABEL: cvtf4_f3:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    retq
 entry:
   %extractVec = shufflevector <4 x float> %x, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %extractVec
diff --git a/test/CodeGen/X86/vec_reassociate.ll b/test/CodeGen/X86/vec_reassociate.ll
index 0d3373528f58..5234b0c8a77c 100644
--- a/test/CodeGen/X86/vec_reassociate.ll
+++ b/test/CodeGen/X86/vec_reassociate.ll
@@ -1,10 +1,17 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X64
 
 define <4 x i32> @add_4i32(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @add_4i32
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   paddd %xmm1, %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: add_4i32:
+; X86:       # BB#0:
+; X86-NEXT:    paddd %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: add_4i32:
+; X64:       # BB#0:
+; X64-NEXT:    paddd %xmm1, %xmm0
+; X64-NEXT:    retq
   %1 = add <4 x i32> %a0, <i32  1, i32 -2, i32  3, i32 -4>
   %2 = add <4 x i32> %a1, <i32 -1, i32  2, i32 -3, i32  4>
   %3 = add <4 x i32> %1, %2
@@ -12,10 +19,15 @@ define <4 x i32> @add_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @add_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @add_4i32_commute
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   paddd %xmm1, %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: add_4i32_commute:
+; X86:       # BB#0:
+; X86-NEXT:    paddd %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: add_4i32_commute:
+; X64:       # BB#0:
+; X64-NEXT:    paddd %xmm1, %xmm0
+; X64-NEXT:    retq
   %1 = add <4 x i32> <i32  1, i32 -2, i32  3, i32 -4>, %a0
   %2 = add <4 x i32> <i32 -1, i32  2, i32 -3, i32  4>, %a1
   %3 = add <4 x i32> %1, %2
@@ -23,11 +35,17 @@ define <4 x i32> @add_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @mul_4i32(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @mul_4i32
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   pmulld %xmm1, %xmm0
-  ;CHECK-NEXT:   pmulld .LCPI2_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: mul_4i32:
+; X86:       # BB#0:
+; X86-NEXT:    pmulld %xmm1, %xmm0
+; X86-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_4i32:
+; X64:       # BB#0:
+; X64-NEXT:    pmulld %xmm1, %xmm0
+; X64-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = mul <4 x i32> %a0, <i32 1, i32 2, i32 3, i32 4>
   %2 = mul <4 x i32> %a1, <i32 4, i32 3, i32 2, i32 1>
   %3 = mul <4 x i32> %1, %2
@@ -35,11 +53,17 @@ define <4 x i32> @mul_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @mul_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @mul_4i32_commute
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   pmulld %xmm1, %xmm0
-  ;CHECK-NEXT:   pmulld .LCPI3_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: mul_4i32_commute:
+; X86:       # BB#0:
+; X86-NEXT:    pmulld %xmm1, %xmm0
+; X86-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_4i32_commute:
+; X64:       # BB#0:
+; X64-NEXT:    pmulld %xmm1, %xmm0
+; X64-NEXT:    pmulld {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = mul <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %a0
   %2 = mul <4 x i32> <i32 4, i32 3, i32 2, i32 1>, %a1
   %3 = mul <4 x i32> %1, %2
@@ -47,11 +71,17 @@ define <4 x i32> @mul_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @and_4i32(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @and_4i32
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   andps %xmm1, %xmm0
-  ;CHECK-NEXT:   andps .LCPI4_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: and_4i32:
+; X86:       # BB#0:
+; X86-NEXT:    andps %xmm1, %xmm0
+; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_4i32:
+; X64:       # BB#0:
+; X64-NEXT:    andps %xmm1, %xmm0
+; X64-NEXT:    andps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = and <4 x i32> %a0, <i32 -2, i32 -2, i32  3, i32  3>
   %2 = and <4 x i32> %a1, <i32 -1, i32 -1, i32  1, i32  1>
   %3 = and <4 x i32> %1, %2
@@ -59,11 +89,17 @@ define <4 x i32> @and_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @and_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @and_4i32_commute
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   andps %xmm1, %xmm0
-  ;CHECK-NEXT:   andps .LCPI5_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: and_4i32_commute:
+; X86:       # BB#0:
+; X86-NEXT:    andps %xmm1, %xmm0
+; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_4i32_commute:
+; X64:       # BB#0:
+; X64-NEXT:    andps %xmm1, %xmm0
+; X64-NEXT:    andps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = and <4 x i32> <i32 -2, i32 -2, i32  3, i32  3>, %a0
   %2 = and <4 x i32> <i32 -1, i32 -1, i32  1, i32  1>, %a1
   %3 = and <4 x i32> %1, %2
@@ -71,11 +107,17 @@ define <4 x i32> @and_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @or_4i32(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @or_4i32
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   orps %xmm1, %xmm0
-  ;CHECK-NEXT:   orps .LCPI6_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: or_4i32:
+; X86:       # BB#0:
+; X86-NEXT:    orps %xmm1, %xmm0
+; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: or_4i32:
+; X64:       # BB#0:
+; X64-NEXT:    orps %xmm1, %xmm0
+; X64-NEXT:    orps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = or <4 x i32> %a0, <i32 -2, i32 -2, i32  3, i32  3>
   %2 = or <4 x i32> %a1, <i32 -1, i32 -1, i32  1, i32  1>
   %3 = or <4 x i32> %1, %2
@@ -83,23 +125,35 @@ define <4 x i32> @or_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @or_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @or_4i32_commute
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   orps %xmm1, %xmm0
-  ;CHECK-NEXT:   orps .LCPI7_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
-  %1 = or <4 x i32> <i32 -2, i32 -2, i32  3, i32  3>, %a0 
+; X86-LABEL: or_4i32_commute:
+; X86:       # BB#0:
+; X86-NEXT:    orps %xmm1, %xmm0
+; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: or_4i32_commute:
+; X64:       # BB#0:
+; X64-NEXT:    orps %xmm1, %xmm0
+; X64-NEXT:    orps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
+  %1 = or <4 x i32> <i32 -2, i32 -2, i32  3, i32  3>, %a0
   %2 = or <4 x i32> <i32 -1, i32 -1, i32  1, i32  1>, %a1
   %3 = or <4 x i32> %1, %2
   ret <4 x i32> %3
 }
 
 define <4 x i32> @xor_4i32(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @xor_4i32
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   xorps %xmm1, %xmm0
-  ;CHECK-NEXT:   xorps .LCPI8_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: xor_4i32:
+; X86:       # BB#0:
+; X86-NEXT:    xorps %xmm1, %xmm0
+; X86-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_4i32:
+; X64:       # BB#0:
+; X64-NEXT:    xorps %xmm1, %xmm0
+; X64-NEXT:    xorps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = xor <4 x i32> %a0, <i32 -2, i32 -2, i32  3, i32  3>
   %2 = xor <4 x i32> %a1, <i32 -1, i32 -1, i32  1, i32  1>
   %3 = xor <4 x i32> %1, %2
@@ -107,11 +161,17 @@ define <4 x i32> @xor_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <4 x i32> @xor_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
-  ;CHECK-LABEL:  @xor_4i32_commute
-  ;CHECK:        # BB#0:
-  ;CHECK-NEXT:   xorps %xmm1, %xmm0
-  ;CHECK-NEXT:   xorps .LCPI9_0(%rip), %xmm0
-  ;CHECK-NEXT:   retq
+; X86-LABEL: xor_4i32_commute:
+; X86:       # BB#0:
+; X86-NEXT:    xorps %xmm1, %xmm0
+; X86-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_4i32_commute:
+; X64:       # BB#0:
+; X64-NEXT:    xorps %xmm1, %xmm0
+; X64-NEXT:    xorps {{.*}}(%rip), %xmm0
+; X64-NEXT:    retq
   %1 = xor <4 x i32> <i32 -2, i32 -2, i32  3, i32  3>, %a0
   %2 = xor <4 x i32> <i32 -1, i32 -1, i32  1, i32  1>, %a1
   %3 = xor <4 x i32> %1, %2
diff --git a/test/CodeGen/X86/vector-lzcnt-512.ll b/test/CodeGen/X86/vector-lzcnt-512.ll
index 79d133bbfb8f..88378eb51a27 100644
--- a/test/CodeGen/X86/vector-lzcnt-512.ll
+++ b/test/CodeGen/X86/vector-lzcnt-512.ll
@@ -1,39 +1,337 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512CD
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd,-avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512CD
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512CDBW
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=-avx512cd,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=-avx512cd,-avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512DQ
 
 define <8 x i64> @testv8i64(<8 x i64> %in) nounwind {
-; ALL-LABEL: testv8i64:
-; ALL:       ## BB#0:
-; ALL-NEXT:    vplzcntq %zmm0, %zmm0
-; ALL-NEXT:    retq
+; AVX512CD-LABEL: testv8i64:
+; AVX512CD:       ## BB#0:
+; AVX512CD-NEXT:    vplzcntq %zmm0, %zmm0
+; AVX512CD-NEXT:    retq
+;
+; AVX512CDBW-LABEL: testv8i64:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vplzcntq %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
+; AVX512BW-LABEL: testv8i64:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vpsrlq $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $32, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv8i64:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vpsrlq $1, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $2, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $4, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $8, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpxor %ymm3, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm5
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm5, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
   %out = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %in, i1 0)
   ret <8 x i64> %out
 }
 
 define <8 x i64> @testv8i64u(<8 x i64> %in) nounwind {
-; ALL-LABEL: testv8i64u:
-; ALL:       ## BB#0:
-; ALL-NEXT:    vplzcntq %zmm0, %zmm0
-; ALL-NEXT:    retq
+; AVX512CD-LABEL: testv8i64u:
+; AVX512CD:       ## BB#0:
+; AVX512CD-NEXT:    vplzcntq %zmm0, %zmm0
+; AVX512CD-NEXT:    retq
+;
+; AVX512CDBW-LABEL: testv8i64u:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vplzcntq %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
+; AVX512BW-LABEL: testv8i64u:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vpsrlq $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlq $32, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv8i64u:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vpsrlq $1, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $2, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $4, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $8, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpxor %ymm3, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm5
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm5, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
   %out = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %in, i1 -1)
   ret <8 x i64> %out
 }
 
 define <16 x i32> @testv16i32(<16 x i32> %in) nounwind {
-; ALL-LABEL: testv16i32:
-; ALL:       ## BB#0:
-; ALL-NEXT:    vplzcntd %zmm0, %zmm0
-; ALL-NEXT:    retq
+; AVX512CD-LABEL: testv16i32:
+; AVX512CD:       ## BB#0:
+; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CD-NEXT:    retq
+;
+; AVX512CDBW-LABEL: testv16i32:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
+; AVX512BW-LABEL: testv16i32:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vpsrld $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
+; AVX512BW-NEXT:    vpsadbw %zmm1, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; AVX512BW-NEXT:    vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv16i32:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vpsrld $1, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $2, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $4, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $8, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $16, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpxor %ymm3, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm5
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm5, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
   %out = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %in, i1 0)
   ret <16 x i32> %out
 }
 
 define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind {
-; ALL-LABEL: testv16i32u:
-; ALL:       ## BB#0:
-; ALL-NEXT:    vplzcntd %zmm0, %zmm0
-; ALL-NEXT:    retq
+; AVX512CD-LABEL: testv16i32u:
+; AVX512CD:       ## BB#0:
+; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CD-NEXT:    retq
+;
+; AVX512CDBW-LABEL: testv16i32u:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
+; AVX512BW-LABEL: testv16i32u:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vpsrld $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
+; AVX512BW-NEXT:    vpsadbw %zmm1, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; AVX512BW-NEXT:    vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv16i32u:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vpsrld $1, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $2, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $4, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $8, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $16, %zmm0, %zmm1
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpxor %ymm3, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm5
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm5, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
+; AVX512DQ-NEXT:    vpsadbw %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
   %out = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %in, i1 -1)
   ret <16 x i32> %out
 }
@@ -52,20 +350,78 @@ define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
 ; AVX512CD-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
 ; AVX512CD-NEXT:    retq
 ;
+; AVX512CDBW-LABEL: testv32i16:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512CDBW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm1, %zmm1
+; AVX512CDBW-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512CDBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512CDBW-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512CDBW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CDBW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512CDBW-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512CDBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
 ; AVX512BW-LABEL: testv32i16:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512BW-NEXT:    vplzcntd %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmovdw %zmm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BW-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512BW-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv32i16:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vpxor %ymm6, %ymm6, %ymm6
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm5, %ymm7
+; AVX512DQ-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm5
+; AVX512DQ-NEXT:    vpaddb %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpaddw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm5
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm5, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    retq
   %out = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %in, i1 0)
   ret <32 x i16> %out
 }
@@ -84,20 +440,78 @@ define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
 ; AVX512CD-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
 ; AVX512CD-NEXT:    retq
 ;
+; AVX512CDBW-LABEL: testv32i16u:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512CDBW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm1, %zmm1
+; AVX512CDBW-NEXT:    vpmovdw %zmm1, %ymm1
+; AVX512CDBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512CDBW-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512CDBW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CDBW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512CDBW-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
+; AVX512CDBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
 ; AVX512BW-LABEL: testv32i16u:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512BW-NEXT:    vplzcntd %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmovdw %zmm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BW-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512BW-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512BW-NEXT:    vpsubw %ymm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandnq %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv32i16u:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vpxor %ymm6, %ymm6, %ymm6
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm5, %ymm7
+; AVX512DQ-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm5
+; AVX512DQ-NEXT:    vpaddb %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpaddw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm5
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm5, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqb %ymm6, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    retq
   %out = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %in, i1 -1)
   ret <32 x i16> %out
 }
@@ -128,32 +542,78 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX512CD-NEXT:    retq
 ;
+; AVX512CDBW-LABEL: testv64i8:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512CDBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm2, %zmm2
+; AVX512CDBW-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CDBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm1, %zmm1
+; AVX512CDBW-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX512CDBW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512CDBW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm2, %zmm2
+; AVX512CDBW-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CDBW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
+; AVX512CDBW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512CDBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
 ; AVX512BW-LABEL: testv64i8:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm2, %zmm2
-; AVX512BW-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm2, %zmm2
-; AVX512BW-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandq %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandnq %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv64i8:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpxor %ymm5, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    retq
   %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 0)
   ret <64 x i8> %out
 }
@@ -184,32 +644,78 @@ define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
 ; AVX512CD-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX512CD-NEXT:    retq
 ;
+; AVX512CDBW-LABEL: testv64i8u:
+; AVX512CDBW:       ## BB#0:
+; AVX512CDBW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512CDBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm2, %zmm2
+; AVX512CDBW-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CDBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm1, %zmm1
+; AVX512CDBW-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX512CDBW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512CDBW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm2, %zmm2
+; AVX512CDBW-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX512CDBW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512CDBW-NEXT:    vplzcntd %zmm0, %zmm0
+; AVX512CDBW-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512CDBW-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
+; AVX512CDBW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512CDBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512CDBW-NEXT:    retq
+;
 ; AVX512BW-LABEL: testv64i8u:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm2, %zmm2
-; AVX512BW-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24]
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm2, %zmm2
-; AVX512BW-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512BW-NEXT:    vplzcntd %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $1, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpandq %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandnq %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu8 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512BW-NEXT:    vpshufb %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpxorq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: testv64i8u:
+; AVX512DQ:       ## BB#0:
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpxor %ymm5, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512DQ-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm2
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    retq
   %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 -1)
   ret <64 x i8> %out
 }
diff --git a/test/CodeGen/X86/vector-shuffle-variable-128.ll b/test/CodeGen/X86/vector-shuffle-variable-128.ll
index 87fd4a7bf6b9..bde8a16d2a5a 100644
--- a/test/CodeGen/X86/vector-shuffle-variable-128.ll
+++ b/test/CodeGen/X86/vector-shuffle-variable-128.ll
@@ -1303,70 +1303,39 @@ define <8 x i16> @var_shuffle_v8i16_v8i16_xyxyxy00_i16(<8 x i16> %x, <8 x i16> %
 ; SSE41-NEXT:    andl $7, %r8d
 ; SSE41-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; SSE41-NEXT:    andl $7, %r9d
-; SSE41-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
-; SSE41-NEXT:    movd %eax, %xmm1
-; SSE41-NEXT:    pinsrw $1, -24(%rsp,%rsi,2), %xmm1
-; SSE41-NEXT:    pinsrw $2, -40(%rsp,%rdx,2), %xmm1
-; SSE41-NEXT:    pinsrw $3, -24(%rsp,%rcx,2), %xmm1
-; SSE41-NEXT:    pinsrw $4, -40(%rsp,%r8,2), %xmm1
-; SSE41-NEXT:    pinsrw $5, -24(%rsp,%r9,2), %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
+; SSE41-NEXT:    pinsrw $0, -40(%rsp,%rdi,2), %xmm0
+; SSE41-NEXT:    pinsrw $1, -24(%rsp,%rsi,2), %xmm0
+; SSE41-NEXT:    pinsrw $2, -40(%rsp,%rdx,2), %xmm0
+; SSE41-NEXT:    pinsrw $3, -24(%rsp,%rcx,2), %xmm0
+; SSE41-NEXT:    pinsrw $4, -40(%rsp,%r8,2), %xmm0
+; SSE41-NEXT:    pinsrw $5, -24(%rsp,%r9,2), %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
-; AVX1-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
-; AVX1-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
-; AVX1-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
-; AVX1-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
-; AVX1-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; AVX1-NEXT:    andl $7, %edi
-; AVX1-NEXT:    andl $7, %esi
-; AVX1-NEXT:    andl $7, %edx
-; AVX1-NEXT:    andl $7, %ecx
-; AVX1-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    andl $7, %r8d
-; AVX1-NEXT:    vmovdqa %xmm1, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    andl $7, %r9d
-; AVX1-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $2, -40(%rsp,%rdx,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $4, -40(%rsp,%r8,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
-; AVX2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
-; AVX2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
-; AVX2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
-; AVX2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
-; AVX2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; AVX2-NEXT:    andl $7, %edi
-; AVX2-NEXT:    andl $7, %esi
-; AVX2-NEXT:    andl $7, %edx
-; AVX2-NEXT:    andl $7, %ecx
-; AVX2-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    andl $7, %r8d
-; AVX2-NEXT:    vmovdqa %xmm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    andl $7, %r9d
-; AVX2-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
-; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $2, -40(%rsp,%rdx,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $4, -40(%rsp,%r8,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
-; AVX2-NEXT:    retq
+; AVX-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; AVX-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; AVX-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; AVX-NEXT:    andl $7, %edi
+; AVX-NEXT:    andl $7, %esi
+; AVX-NEXT:    andl $7, %edx
+; AVX-NEXT:    andl $7, %ecx
+; AVX-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    andl $7, %r8d
+; AVX-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
+; AVX-NEXT:    andl $7, %r9d
+; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $0, -40(%rsp,%rdi,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $2, -40(%rsp,%rdx,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $4, -40(%rsp,%r8,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %x0 = extractelement <8 x i16> %x, i16 %i0
   %y1 = extractelement <8 x i16> %y, i16 %i1
   %x2 = extractelement <8 x i16> %x, i16 %i2
diff --git a/test/CodeGen/X86/win64_eh_leaf.ll b/test/CodeGen/X86/win64_eh_leaf.ll
index 21a423ab36a9..35d55a907375 100644
--- a/test/CodeGen/X86/win64_eh_leaf.ll
+++ b/test/CodeGen/X86/win64_eh_leaf.ll
@@ -29,3 +29,12 @@ entry:
 ; and no unwind info in the object file.
 ; READOBJ-NOT: leaf_func
 }
+
+define void @naked_func() naked {
+  call void asm sideeffect "ret", ""()
+  unreachable
+}
+; ASM-LABEL: naked_func:
+; ASM-NOT: .seh_
+; ASM: ret
+; ASM-NOT: .seh_
diff --git a/test/CodeGen/X86/xray-attribute-instrumentation.ll b/test/CodeGen/X86/xray-attribute-instrumentation.ll
index c52ccf9356bc..7c60327d2c30 100644
--- a/test/CodeGen/X86/xray-attribute-instrumentation.ll
+++ b/test/CodeGen/X86/xray-attribute-instrumentation.ll
@@ -15,10 +15,17 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 }
 ; CHECK:       .p2align 4, 0x90
 ; CHECK-NEXT:  .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_0
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .quad {{.*}}xray_sled_0
 ; CHECK:       .quad {{.*}}xray_sled_1
+; CHECK-LABEL: Lxray_synthetic_end0:
+; CHECK:       .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_0:
+; CHECK:       .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_end0
+
 
 ; We test multiple returns in a single function to make sure we're getting all
 ; of them with XRay instrumentation.
@@ -46,8 +53,14 @@ NotEqual:
 }
 ; CHECK:       .p2align 4, 0x90
 ; CHECK-NEXT:  .quad {{.*}}xray_synthetic_1
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_1
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map
 ; CHECK-LABEL: Lxray_synthetic_1:
 ; CHECK:       .quad {{.*}}xray_sled_2
 ; CHECK:       .quad {{.*}}xray_sled_3
 ; CHECK:       .quad {{.*}}xray_sled_4
+; CHECK-LABEL: Lxray_synthetic_end1:
+; CHECK:       .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_1:
+; CHECK:       .quad {{.*}}xray_synthetic_1
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_end1
diff --git a/test/CodeGen/X86/xray-custom-log.ll b/test/CodeGen/X86/xray-custom-log.ll
new file mode 100644
index 000000000000..63625d44b4cb
--- /dev/null
+++ b/test/CodeGen/X86/xray-custom-log.ll
@@ -0,0 +1,23 @@
+; RUN: llc -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
+    %eventptr = alloca i8
+    %eventsize = alloca i32
+    store i32 3, i32* %eventsize
+    %val = load i32, i32* %eventsize
+    call void @llvm.xray.customevent(i8* %eventptr, i32 %val)
+    ; CHECK-LABEL: Lxray_event_sled_0:
+    ; CHECK-NEXT:  .ascii "\353\024
+    ; CHECK-NEXT:  pushq %rax
+    ; CHECK-NEXT:  movq {{.*}}, %rdi
+    ; CHECK-NEXT:  movq {{.*}}, %rsi
+    ; CHECK-NEXT:  movabsq $__xray_CustomEvent, %rax
+    ; CHECK-NEXT:  callq *%rax
+    ; CHECK-NEXT:  popq %rax
+    ret i32 0
+}
+; CHECK:       .section {{.*}}xray_instr_map
+; CHECK-LABEL: Lxray_synthetic_0:
+; CHECK:       .quad {{.*}}xray_event_sled_0
+
+declare void @llvm.xray.customevent(i8*, i32)
diff --git a/test/CodeGen/X86/xray-loop-detection.ll b/test/CodeGen/X86/xray-loop-detection.ll
new file mode 100644
index 000000000000..3cd6b4aa6f8c
--- /dev/null
+++ b/test/CodeGen/X86/xray-loop-detection.ll
@@ -0,0 +1,23 @@
+; RUN: llc -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -filetype=asm -o - -mtriple=x86_64-darwin-unknown    < %s | FileCheck %s
+
+define i32 @foo(i32 %i) nounwind noinline uwtable "xray-instruction-threshold"="1" {
+entry:
+  br label %Test
+Test:
+  %indvar = phi i32 [0, %entry], [%nextindvar, %Inc]
+  %cond = icmp eq i32 %indvar, %i
+  br i1 %cond, label %Exit, label %Inc
+Inc:
+  %nextindvar = add i32 %indvar, 1
+  br label %Test
+Exit:
+  %retval = phi i32 [%indvar, %Test]
+  ret i32 %retval
+}
+
+; CHECK-LABEL: xray_sled_0:
+; CHECK-NEXT:  .ascii "\353\t"
+; CHECK-NEXT:  nopw 512(%rax,%rax)
+; CHECK-LABEL: Ltmp0:
+
diff --git a/test/CodeGen/X86/xray-tail-call-sled.ll b/test/CodeGen/X86/xray-tail-call-sled.ll
index ece786a5e809..b12c78a77b20 100644
--- a/test/CodeGen/X86/xray-tail-call-sled.ll
+++ b/test/CodeGen/X86/xray-tail-call-sled.ll
@@ -14,11 +14,17 @@ define i32 @callee() nounwind noinline uwtable "function-instrument"="xray-alway
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
 }
 ; CHECK:       .p2align 4, 0x90
-; CHECK-NEXT:  .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_0{{.*}}
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_0{{.*}}
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .quad {{.*}}xray_sled_0
 ; CHECK:       .quad {{.*}}xray_sled_1
+; CHECK-LABEL: Lxray_synthetic_end0:
+; CHECK-NEXT:  .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_0:
+; CHECK:       .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_end0
 
 define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-always" {
 ; CHECK:       .p2align 1, 0x90
@@ -36,7 +42,13 @@ define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-alway
   ret i32 %retval
 }
 ; CHECK:       .p2align 4, 0x90
-; CHECK-NEXT:  .quad {{.*}}xray_synthetic_1
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_1{{.*}}
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_1{{.*}}
 ; CHECK-LABEL: Lxray_synthetic_1:
 ; CHECK:       .quad {{.*}}xray_sled_2
 ; CHECK:       .quad {{.*}}xray_sled_3
+; CHECK-LABEL: Lxray_synthetic_end1:
+; CHECK:       .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_1:
+; CHECK:       .quad {{.*}}xray_synthetic_1
+; CHECK:       .quad {{.*}}xray_synthetic_end1