vendor/llvm/llvm-trunk-r239412

author: Dimitry Andric <dim@FreeBSD.org> 2015-06-09 19:06:30 +0000
committer: Dimitry Andric <dim@FreeBSD.org> 2015-06-09 19:06:30 +0000
commit: 85d8b2bbe386bcfe669575d05b61482d7be07e5d (patch)
tree: 1dc5e75ab222a9ead44c699eceafab7a6ca7b310 /test/CodeGen/X86
parent: 5a5ac124e1efaf208671f01c46edb15f29ed2a0b (diff)
38 files changed, 9450 insertions, 739 deletions
diff --git a/test/CodeGen/X86/asm-reject-reg-type-mismatch.ll b/test/CodeGen/X86/asm-reject-reg-type-mismatch.ll
new file mode 100644
index 0000000000000..016e2d261eef6
--- /dev/null
+++ b/test/CodeGen/X86/asm-reject-reg-type-mismatch.ll
@@ -0,0 +1,10 @@
+; RUN: not llc -no-integrated-as %s -o - 2> %t1
+; RUN: FileCheck %s < %t1
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64--"
+
+; CHECK: error: couldn't allocate output register for constraint '{ax}'
+define i128 @blup() {
+  %v = tail call i128 asm "", "={ax},0,~{dirflag},~{fpsr},~{flags}"(i128 0)
+  ret i128 %v
+}
diff --git a/test/CodeGen/X86/avx-vperm2x128.ll b/test/CodeGen/X86/avx-vperm2x128.ll
index 10ed079a264e8..74d20f348b529 100644
--- a/test/CodeGen/X86/avx-vperm2x128.ll
+++ b/test/CodeGen/X86/avx-vperm2x128.ll
@@ -147,8 +147,8 @@ define <16 x i16> @E5i(<16 x i16>* %a, <16 x i16>* %b) nounwind uwtable readnone
 ; AVX1-LABEL: E5i:
 ; AVX1:       ## BB#0: ## %entry
 ; AVX1-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vmovaps (%rsi), %ymm1
+; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/avx2-vector-shifts.ll b/test/CodeGen/X86/avx2-vector-shifts.ll
index 8aae90c3c03db..5d99269ae1dc8 100644
--- a/test/CodeGen/X86/avx2-vector-shifts.ll
+++ b/test/CodeGen/X86/avx2-vector-shifts.ll
@@ -300,6 +300,56 @@ define <16 x i16> @shl_16i16(<16 x i16> %r, <16 x i16> %a) nounwind {
   ret <16 x i16> %shl
 }
 
+define <32 x i8> @shl_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
+; CHECK-LABEL:  shl_32i8
+; CHECK:        vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT:   vpsllw $4, %xmm3, %xmm2
+; CHECK-NEXT:   vmovdqa  {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; CHECK-NEXT:   vpand %xmm8, %xmm2, %xmm5
+; CHECK-NEXT:   vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT:   vpsllw $5, %xmm2, %xmm2
+; CHECK-NEXT:   vmovdqa  {{.*#+}} xmm9 = [224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224]
+; CHECK-NEXT:   vpand %xmm9, %xmm2, %xmm7
+; CHECK-NEXT:   vmovdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; CHECK-NEXT:   vpand %xmm7, %xmm2, %xmm4
+; CHECK-NEXT:   vpcmpeqb %xmm2, %xmm4, %xmm4
+; CHECK-NEXT:   vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
+; CHECK-NEXT:   vpsllw $2, %xmm3, %xmm4
+; CHECK-NEXT:   vmovdqa  {{.*#+}} xmm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; CHECK-NEXT:   vpand %xmm5, %xmm4, %xmm4
+; CHECK-NEXT:   vpaddb %xmm7, %xmm7, %xmm7
+; CHECK-NEXT:   vpand %xmm7, %xmm2, %xmm6
+; CHECK-NEXT:   vpcmpeqb %xmm2, %xmm6, %xmm6
+; CHECK-NEXT:   vpblendvb %xmm6, %xmm4, %xmm3, %xmm3
+; CHECK-NEXT:   vpaddb %xmm3, %xmm3, %xmm4
+; CHECK-NEXT:   vpaddb %xmm7, %xmm7, %xmm6
+; CHECK-NEXT:   vpand %xmm6, %xmm2, %xmm6
+; CHECK-NEXT:   vpcmpeqb %xmm2, %xmm6, %xmm6
+; CHECK-NEXT:   vpblendvb %xmm6, %xmm4, %xmm3, %xmm3
+; CHECK-NEXT:   vpsllw $4, %xmm0, %xmm4
+; CHECK-NEXT:   vpand %xmm8, %xmm4, %xmm4
+; CHECK-NEXT:   vpsllw $5, %xmm1, %xmm1
+; CHECK-NEXT:   vpand %xmm9, %xmm1, %xmm1
+; CHECK-NEXT:   vpand %xmm1, %xmm2, %xmm6
+; CHECK-NEXT:   vpcmpeqb %xmm2, %xmm6, %xmm6
+; CHECK-NEXT:   vpblendvb %xmm6, %xmm4, %xmm0, %xmm0
+; CHECK-NEXT:   vpsllw $2, %xmm0, %xmm4
+; CHECK-NEXT:   vpand %xmm5, %xmm4, %xmm4
+; CHECK-NEXT:   vpaddb %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:   vpand %xmm1, %xmm2, %xmm5
+; CHECK-NEXT:   vpcmpeqb %xmm2, %xmm5, %xmm5
+; CHECK-NEXT:   vpblendvb %xmm5, %xmm4, %xmm0, %xmm0
+; CHECK-NEXT:   vpaddb %xmm0, %xmm0, %xmm4
+; CHECK-NEXT:   vpaddb %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:   vpand %xmm1, %xmm2, %xmm1
+; CHECK-NEXT:   vpcmpeqb %xmm2, %xmm1, %xmm1
+; CHECK-NEXT:   vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; CHECK-NEXT:   vinserti128 $1, %xmm3, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %shl = shl <32 x i8> %r, %a
+  ret <32 x i8> %shl
+}
+
 define <8 x i16> @ashr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
 ; CHECK-LABEL:  ashr_8i16
 ; CHECK:        vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
@@ -329,6 +379,176 @@ define <16 x i16> @ashr_16i16(<16 x i16> %r, <16 x i16> %a) nounwind {
   ret <16 x i16> %ashr
 }
 
+define <32 x i8> @ashr_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
+; CHECK-LABEL:  ashr_32i8
+; CHECK:        vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT:   vpextrb $1, %xmm2, %ecx
+; CHECK-NEXT:   vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT:   vpextrb $1, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $0, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $0, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   movzbl %dl, %edx
+; CHECK-NEXT:   vpextrb $2, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $2, %xmm3, %esi
+; CHECK-NEXT:   sarb %cl, %sil
+; CHECK-NEXT:   vmovd %edx, %xmm4
+; CHECK-NEXT:   vpinsrb $1, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %sil, %eax
+; CHECK-NEXT:   vpextrb $3, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $3, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $3, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $4, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $4, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $4, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $5, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $5, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $6, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $6, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $5, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $7, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $7, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $6, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $7, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $8, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $8, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $8, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $9, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $9, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $10, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $10, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $9, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $11, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $11, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $10, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $11, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $12, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $12, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $12, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $13, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $13, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $14, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $14, %xmm3, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $13, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $15, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $15, %xmm3, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $1, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $1, %xmm0, %esi
+; CHECK-NEXT:   sarb %cl, %sil
+; CHECK-NEXT:   movzbl %dl, %ecx
+; CHECK-NEXT:   vpinsrb $14, %ecx, %xmm4, %xmm2
+; CHECK-NEXT:   vpextrb $0, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $0, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpextrb $2, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $2, %xmm0, %edi
+; CHECK-NEXT:   sarb %cl, %dil
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $15, %eax, %xmm2, %xmm2
+; CHECK-NEXT:   movzbl %sil, %eax
+; CHECK-NEXT:   movzbl %dl, %ecx
+; CHECK-NEXT:   vmovd %ecx, %xmm3
+; CHECK-NEXT:   vpinsrb $1, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dil, %eax
+; CHECK-NEXT:   vpextrb $3, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $3, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $2, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $3, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $4, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $4, %xmm0, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $4, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $5, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $5, %xmm0, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $6, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $6, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $5, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $7, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $7, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $6, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $7, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $8, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $8, %xmm0, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $8, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $9, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $9, %xmm0, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $10, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $10, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $9, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $11, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $11, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $10, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $11, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $12, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $12, %xmm0, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $12, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $13, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $13, %xmm0, %eax
+; CHECK-NEXT:   sarb %cl, %al
+; CHECK-NEXT:   vpextrb $14, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $14, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $13, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $15, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $15, %xmm0, %edx
+; CHECK-NEXT:   sarb %cl, %dl
+; CHECK-NEXT:   vpinsrb $14, %eax, %xmm3, %xmm0
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $15, %eax, %xmm0, %xmm0
+; CHECK-NEXT:   vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %ashr = ashr <32 x i8> %r, %a
+  ret <32 x i8> %ashr
+}
+
 define <8 x i16> @lshr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind {
 ; CHECK-LABEL:  lshr_8i16
 ; CHECK:        vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
@@ -357,3 +577,173 @@ define <16 x i16> @lshr_16i16(<16 x i16> %r, <16 x i16> %a) nounwind {
   %lshr = lshr <16 x i16> %r, %a
   ret <16 x i16> %lshr
 }
+
+define <32 x i8> @lshr_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
+; CHECK-LABEL:  lshr_32i8
+; CHECK:        vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT:   vpextrb $1, %xmm2, %ecx
+; CHECK-NEXT:   vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT:   vpextrb $1, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $0, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $0, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   movzbl %dl, %edx
+; CHECK-NEXT:   vpextrb $2, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $2, %xmm3, %esi
+; CHECK-NEXT:   shrb %cl, %sil
+; CHECK-NEXT:   vmovd %edx, %xmm4
+; CHECK-NEXT:   vpinsrb $1, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %sil, %eax
+; CHECK-NEXT:   vpextrb $3, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $3, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $3, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $4, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $4, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $4, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $5, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $5, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $6, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $6, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $5, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $7, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $7, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $6, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $7, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $8, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $8, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $8, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $9, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $9, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $10, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $10, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $9, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $11, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $11, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $10, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $11, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $12, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $12, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $12, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $13, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $13, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $14, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $14, %xmm3, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $13, %eax, %xmm4, %xmm4
+; CHECK-NEXT:   vpextrb $15, %xmm2, %ecx
+; CHECK-NEXT:   vpextrb $15, %xmm3, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $1, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $1, %xmm0, %esi
+; CHECK-NEXT:   shrb %cl, %sil
+; CHECK-NEXT:   movzbl %dl, %ecx
+; CHECK-NEXT:   vpinsrb $14, %ecx, %xmm4, %xmm2
+; CHECK-NEXT:   vpextrb $0, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $0, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpextrb $2, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $2, %xmm0, %edi
+; CHECK-NEXT:   shrb %cl, %dil
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $15, %eax, %xmm2, %xmm2
+; CHECK-NEXT:   movzbl %sil, %eax
+; CHECK-NEXT:   movzbl %dl, %ecx
+; CHECK-NEXT:   vmovd %ecx, %xmm3
+; CHECK-NEXT:   vpinsrb $1, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dil, %eax
+; CHECK-NEXT:   vpextrb $3, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $3, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $2, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $3, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $4, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $4, %xmm0, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $4, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $5, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $5, %xmm0, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $6, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $6, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $5, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $7, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $7, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $6, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $7, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $8, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $8, %xmm0, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $8, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $9, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $9, %xmm0, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $10, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $10, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $9, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $11, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $11, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $10, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $11, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $12, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $12, %xmm0, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $12, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   vpextrb $13, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $13, %xmm0, %eax
+; CHECK-NEXT:   shrb %cl, %al
+; CHECK-NEXT:   vpextrb $14, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $14, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   movzbl %al, %eax
+; CHECK-NEXT:   vpinsrb $13, %eax, %xmm3, %xmm3
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpextrb $15, %xmm1, %ecx
+; CHECK-NEXT:   vpextrb $15, %xmm0, %edx
+; CHECK-NEXT:   shrb %cl, %dl
+; CHECK-NEXT:   vpinsrb $14, %eax, %xmm3, %xmm0
+; CHECK-NEXT:   movzbl %dl, %eax
+; CHECK-NEXT:   vpinsrb $15, %eax, %xmm0, %xmm0
+; CHECK-NEXT:   vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT:   retq
+  %lshr = lshr <32 x i8> %r, %a
+  ret <32 x i8> %lshr
+}
diff --git a/test/CodeGen/X86/avx512-build-vector.ll b/test/CodeGen/X86/avx512-build-vector.ll
index 8373c6da26199..e70d9f3ad521c 100644
--- a/test/CodeGen/X86/avx512-build-vector.ll
+++ b/test/CodeGen/X86/avx512-build-vector.ll
@@ -2,13 +2,9 @@
 
 define <16 x i32> @test1(i32* %x) {
 ; CHECK-LABEL: test1:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovd (%rdi), %xmm0
-; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    vpxor %ymm1, %ymm1, %ymm1
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7]
-; CHECK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK:    vmovd (%rdi), %xmm
+; CHECK:    vmovdqa32
+; CHECK:    vpermt2d %zmm
    %y = load i32, i32* %x, align 4
    %res = insertelement <16 x i32>zeroinitializer, i32 %y, i32 4
    ret <16 x i32>%res
diff --git a/test/CodeGen/X86/avx512-intrinsics.ll b/test/CodeGen/X86/avx512-intrinsics.ll
index 471e34cdedced..9387192f8aa44 100644
--- a/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/test/CodeGen/X86/avx512-intrinsics.ll
@@ -98,18 +98,55 @@ define <4 x float> @test_rcp14_ss(<4 x float> %a0) {
 declare <4 x float> @llvm.x86.avx512.rcp14.ss(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
 define <8 x double> @test_sqrt_pd_512(<8 x double> %a0) {
+  ; CHECK-LABEL: test_sqrt_pd_512
   ; CHECK: vsqrtpd
-  %res = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4) ; <<8 x double>> [#uses=1]
+  %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4) 
   ret <8 x double> %res
 }
-declare <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
+declare <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
 
 define <16 x float> @test_sqrt_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_sqrt_ps_512
   ; CHECK: vsqrtps
-  %res = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) ; <<16 x float>> [#uses=1]
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) 
   ret <16 x float> %res
 }
-declare <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+define <16 x float> @test_sqrt_round_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_sqrt_round_ps_512
+  ; CHECK: vsqrtps {rz-sae}
+  %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 3) 
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
+
+define <8 x double> @test_getexp_pd_512(<8 x double> %a0) {
+  ; CHECK-LABEL: test_getexp_pd_512
+  ; CHECK: vgetexppd
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4) 
+  ret <8 x double> %res
+}
+define <8 x double> @test_getexp_round_pd_512(<8 x double> %a0) {
+  ; CHECK-LABEL: test_getexp_round_pd_512
+  ; CHECK: vgetexppd {sae}
+  %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 8) 
+  ret <8 x double> %res
+}
+declare <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone
+
+define <16 x float> @test_getexp_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_getexp_ps_512
+  ; CHECK: vgetexpps
+  %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) 
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_getexp_round_ps_512(<16 x float> %a0) {
+  ; CHECK-LABEL: test_getexp_round_ps_512
+  ; CHECK: vgetexpps {sae}
+  %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8) 
+  ret <16 x float> %res
+}
+declare <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone
 
 define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1) {
   ; CHECK: vsqrtss {{.*}}encoding: [0x62
diff --git a/test/CodeGen/X86/avx512-shuffle.ll b/test/CodeGen/X86/avx512-shuffle.ll
new file mode 100644
index 0000000000000..2683d6fe238c5
--- /dev/null
+++ b/test/CodeGen/X86/avx512-shuffle.ll
@@ -0,0 +1,336 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=CHECK-SKX
+
+; CHECK-LABEL: test1:
+; CHECK: vpermps
+; CHECK: ret
+define <16 x float> @test1(<16 x float> %a) nounwind {
+  %c = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> <i32 2, i32 5, i32 undef, i32 undef, i32 7, i32 undef, i32 10, i32 1,  i32 0, i32 5, i32 undef, i32 4, i32 7, i32 undef, i32 10, i32 1>
+  ret <16 x float> %c
+}
+
+; CHECK-LABEL: test2:
+; CHECK: vpermd
+; CHECK: ret
+define <16 x i32> @test2(<16 x i32> %a) nounwind {
+  %c = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 2, i32 5, i32 undef, i32 undef, i32 7, i32 undef, i32 10, i32 1,  i32 0, i32 5, i32 undef, i32 4, i32 7, i32 undef, i32 10, i32 1>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: test3:
+; CHECK: vpermq
+; CHECK: ret
+define <8 x i64> @test3(<8 x i64> %a) nounwind {
+  %c = shufflevector <8 x i64> %a, <8 x i64> undef, <8 x i32> <i32 2, i32 5, i32 1, i32 undef, i32 7, i32 undef, i32 3, i32 1>
+  ret <8 x i64> %c
+}
+
+; CHECK-LABEL: test4:
+; CHECK: vpermpd
+; CHECK: ret
+define <8 x double> @test4(<8 x double> %a) nounwind {
+  %c = shufflevector <8 x double> %a, <8 x double> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x double> %c
+}
+
+; CHECK-LABEL: test5:
+; CHECK: vpermt2pd
+; CHECK: ret
+define <8 x double> @test5(<8 x double> %a, <8 x double> %b) nounwind {
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 2, i32 8, i32 0, i32 1, i32 6, i32 10, i32 4, i32 5>
+  ret <8 x double> %c
+}
+
+; CHECK-LABEL: test6:
+; CHECK: vpermq $30
+; CHECK: ret
+define <8 x i64> @test6(<8 x i64> %a) nounwind {
+  %c = shufflevector <8 x i64> %a, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 1, i32 0, i32 6, i32 7, i32 5, i32 4>
+  ret <8 x i64> %c
+}
+
+; CHECK-LABEL: test7:
+; CHECK: vpermt2q
+; CHECK: ret
+define <8 x i64> @test7(<8 x i64> %a, <8 x i64> %b) nounwind {
+  %c = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 2, i32 8, i32 0, i32 1, i32 6, i32 10, i32 4, i32 5>
+  ret <8 x i64> %c
+}
+
+; CHECK-LABEL: test8:
+; CHECK: vpermt2d
+; CHECK: ret
+define <16 x i32> @test8(<16 x i32> %a, <16 x i32> %b) nounwind {
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: test9:
+; CHECK: vpermt2ps
+; CHECK: ret
+define <16 x float> @test9(<16 x float> %a, <16 x float> %b) nounwind {
+  %c = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
+  ret <16 x float> %c
+}
+
+; CHECK-LABEL: test10:
+; CHECK: vpermt2ps (
+; CHECK: ret
+define <16 x float> @test10(<16 x float> %a, <16 x float>* %b) nounwind {
+  %c = load <16 x float>, <16 x float>* %b
+  %d = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
+  ret <16 x float> %d
+}
+
+; CHECK-LABEL: test11:
+; CHECK: vpermt2d 
+; CHECK: ret
+define <16 x i32> @test11(<16 x i32> %a, <16 x i32>* %b) nounwind {
+  %c = load <16 x i32>, <16 x i32>* %b
+  %d = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
+  ret <16 x i32> %d
+}
+
+; CHECK-LABEL: test13
+; CHECK: vpermilps $177, %zmm
+; CHECK: ret
+define <16 x float> @test13(<16 x float> %a) {
+ %b = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32><i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ ret <16 x float> %b
+}
+
+; CHECK-LABEL: test14
+; CHECK: vpermilpd $203, %zmm
+; CHECK: ret
+define <8 x double> @test14(<8 x double> %a) {
+ %b = shufflevector <8 x double> %a, <8 x double> undef, <8 x i32><i32 1, i32 1, i32 2, i32 3, i32 4, i32 4, i32 7, i32 7>
+ ret <8 x double> %b
+}
+
+; CHECK-LABEL: test15
+; CHECK: vpshufd $177, %zmm
+; CHECK: ret
+define <16 x i32> @test15(<16 x i32> %a) {
+; mask 1-0-3-2 = 10110001 = 0xb1 = 177
+ %b = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32><i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ ret <16 x i32> %b
+}
+; CHECK-LABEL: test16
+; CHECK: valignq $2, %zmm0, %zmm1
+; CHECK: ret
+define <8 x double> @test16(<8 x double> %a, <8 x double> %b) nounwind {
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+  ret <8 x double> %c
+}
+
+; CHECK-LABEL: test17
+; CHECK: vshufpd $19, %zmm1, %zmm0
+; CHECK: ret
+define <8 x double> @test17(<8 x double> %a, <8 x double> %b) nounwind {
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 9, i32 2, i32 10, i32 5, i32 undef, i32 undef, i32 undef>
+  ret <8 x double> %c
+}
+
+; CHECK-LABEL: test18
+; CHECK: vpunpckhdq %zmm
+; CHECK: ret
+define <16 x i32> @test18(<16 x i32> %a, <16 x i32> %c) {
+ %b = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32><i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
+ ret <16 x i32> %b
+}
+
+; CHECK-LABEL: test19
+; CHECK: vpunpckldq %zmm
+; CHECK: ret
+define <16 x i32> @test19(<16 x i32> %a, <16 x i32> %c) {
+ %b = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32><i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+ ret <16 x i32> %b
+}
+
+; CHECK-LABEL: test20
+; CHECK: vpunpckhqdq  %zmm
+; CHECK: ret
+define <8 x i64> @test20(<8 x i64> %a, <8 x i64> %c) {
+ %b = shufflevector <8 x i64> %a, <8 x i64> %c, <8 x i32><i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+ ret <8 x i64> %b
+}
+
+; CHECK-LABEL: test21
+; CHECK: vbroadcastsd  %xmm0, %zmm
+; CHECK: ret
+define <8 x double> @test21(<8 x double> %a, <8 x double> %b) {
+  %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  ret <8 x double> %shuffle
+}
+
+; CHECK-LABEL: test22
+; CHECK: vpbroadcastq  %xmm0, %zmm
+; CHECK: ret
+define <8 x i64> @test22(<8 x i64> %a, <8 x i64> %b) {
+  %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  ret <8 x i64> %shuffle
+}
+
+; CHECK-LABEL: @test23
+; CHECK: vshufps
+; CHECK: vshufps
+; CHECK: ret
+define <16 x i32> @test23(<16 x i32> %a, <16 x i32> %b) nounwind {
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: @test24
+; CHECK: vpermt2d
+; CHECK: ret
+define <16 x i32> @test24(<16 x i32> %a, <16 x i32> %b) nounwind {
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 25, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: @test25
+; CHECK: vshufps  $52
+; CHECK: ret
+define <16 x i32> @test25(<16 x i32> %a, <16 x i32> %b) nounwind {
+; mask - 0-1-3-0 00110100 = 0x34 = 52
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 19, i32 16, i32 4, i32 5, i32 23, i32 undef, i32 8, i32 9, i32 27, i32 undef, i32 12, i32 13, i32 undef, i32 undef>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: @test26
+; CHECK: vmovshdup
+; CHECK: ret
+define <16 x i32> @test26(<16 x i32> %a) nounwind {
+  %c = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 undef, i32 9, i32 9, i32 undef, i32 11, i32 13, i32 undef, i32 undef, i32 undef>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: @test27
+; CHECK: ret
+define <16 x i32> @test27(<4 x i32>%a) {
+ %res = shufflevector <4 x i32> %a, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ ret <16 x i32> %res
+}
+
+; CHECK-LABEL: test28
+; CHECK: vpshufhw $177, %ymm
+; CHECK: ret
+define <16 x i16> @test28(<16 x i16> %a) {
+ %b = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32><i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 7, i32 6, i32 8, i32 9, i32 10, i32 11, i32 13, i32 12, i32 15, i32 14>
+ ret <16 x i16> %b
+}
+
+; CHECK-LABEL: test29
+; CHECK: vunpcklps %zmm
+; CHECK: ret
+define <16 x float> @test29(<16 x float> %a, <16 x float> %c) {
+ %b = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32><i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
+ ret <16 x float> %b
+}
+
+; CHECK-LABEL: @test30
+; CHECK: vshufps $144, %zmm
+; CHECK: ret
+define <16 x float> @test30(<16 x float> %a, <16 x float> %c) {
+ %b = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32><i32 0, i32 0, i32 17, i32 18, i32 4, i32 4, i32 21, i32 22, i32 8, i32 8, i32 25, i32 26, i32 12, i32 12, i32 29, i32 30>
+ ret <16 x float> %b
+}
+
+; CHECK-LABEL: test31
+; CHECK: valignd $3, %zmm0, %zmm1
+; CHECK: ret
+define <16 x i32> @test31(<16 x i32> %a, <16 x i32> %b) nounwind {
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 3, i32 4, i32 5, i32 undef, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+  ret <16 x i32> %c
+}
+
+; CHECK-LABEL: test32
+; CHECK: vshufpd $99, %zmm0, %zmm1
+; CHECK: ret
+define <8 x double> @test32(<8 x double> %a, <8 x double> %b) nounwind {
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 9, i32 1, i32 10, i32 2, i32 undef, i32 5, i32 15, i32 undef>
+  ret <8 x double> %c
+}
+
+define <16 x i32> @test_align_v16i32_rr(<16 x i32> %a, <16 x i32> %b) nounwind {
+; CHECK-LABEL: test_align_v16i32_rr:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    valignd $3, %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 3, i32 4, i32 5, i32 undef, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @test_align_v16i32_rm(<16 x i32>* %a.ptr, <16 x i32> %b) nounwind {
+; CHECK-LABEL: test_align_v16i32_rm:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    valignd $3, (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %a.ptr
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 3, i32 4, i32 5, i32 undef, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @test_align_v16i32_rm_mask(<16 x i32>* %a.ptr, <16 x i32> %b, <16 x i1> %mask) nounwind {
+; CHECK-LABEL: test_align_v16i32_rm_mask:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %zmm1
+; CHECK-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm1, %zmm1
+; CHECK-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; CHECK-NEXT:    vmovdqa32 (%rdi), %zmm1
+; CHECK-NEXT:    valignd $3, %zmm1, %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
+;
+; CHECK-SKX-LABEL: test_align_v16i32_rm_mask:
+; CHECK-SKX:       ## BB#0:
+; CHECK-SKX-NEXT:    vpmovb2m %xmm1, %k1
+; CHECK-SKX-NEXT:    vmovdqa32 (%rdi), %zmm1
+; CHECK-SKX-NEXT:    valignd $3, %zmm1, %zmm0, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-SKX-NEXT:    retq
+  %a = load <16 x i32>, <16 x i32>* %a.ptr
+  %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 3, i32 4, i32 5, i32 undef, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+  %res = select <16 x i1> %mask,<16 x i32> %c, <16 x i32> %a
+  ret <16 x i32> %res
+}
+
+define <8 x double> @test_align_v8f64_rr(<8 x double> %a, <8 x double> %b) nounwind {
+; CHECK-LABEL: test_align_v8f64_rr:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    valignq $3, %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
+  ret <8 x double> %c
+}
+
+define <8 x double> @test_align_v18f64_rm(<8 x double>* %a.ptr, <8 x double> %b) nounwind {
+; CHECK-LABEL: test_align_v18f64_rm:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    valignq $3, (%rdi), %zmm0, %zmm0
+; CHECK-NEXT:    retq
+  %a = load <8 x double>, <8 x double>* %a.ptr
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
+  ret <8 x double> %c
+}
+
+define <8 x double> @test_align_v18f64_rm_mask(<8 x double>* %a.ptr, <8 x double> %b, <8 x i1> %mask) nounwind {
+; CHECK-LABEL: test_align_v18f64_rm_mask:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vpmovsxwq %xmm1, %zmm1
+; CHECK-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm1, %zmm1
+; CHECK-NEXT:    vptestmq %zmm1, %zmm1, %k1
+; CHECK-NEXT:    valignq $3, (%rdi), %zmm0, %zmm0 {%k1} {z}
+; CHECK-NEXT:    retq
+;
+; CHECK-SKX-LABEL: test_align_v18f64_rm_mask:
+; CHECK-SKX:       ## BB#0:
+; CHECK-SKX-NEXT:    vpmovw2m %xmm1, %k1
+; CHECK-SKX-NEXT:    valignq $3, (%rdi), %zmm0, %zmm0 {%k1} {z}
+; CHECK-SKX-NEXT:    retq
+  %a = load <8 x double>, <8 x double>* %a.ptr
+  %c = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
+  %res = select <8 x i1> %mask,<8 x double> %c, <8 x double> zeroinitializer
+  ret <8 x double> %res
+}
+
diff --git a/test/CodeGen/X86/avx512-vec-cmp.ll b/test/CodeGen/X86/avx512-vec-cmp.ll
index e1f6276c6ef4f..04028a1da510d 100644
--- a/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -116,11 +116,8 @@ define <2 x double> @test8(<2 x double> %a, <2 x double> %b) {
 define <8 x i32> @test9(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; KNL-LABEL: test9:
 ; KNL:       ## BB#0:
-; KNL-NEXT:      ## kill: YMM1<def> YMM1<kill> ZMM1<def>
-; KNL-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<def>
 ; KNL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
 ; KNL-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
-; KNL-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<kill>
 ; KNL-NEXT:    retq
   %mask = icmp eq <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %y
@@ -130,11 +127,8 @@ define <8 x i32> @test9(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <8 x float> @test10(<8 x float> %x, <8 x float> %y) nounwind {
 ; KNL-LABEL: test10:
 ; KNL:       ## BB#0:
-; KNL-NEXT:      ## kill: YMM1<def> YMM1<kill> ZMM1<def>
-; KNL-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<def>
 ; KNL-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
 ; KNL-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
-; KNL-NEXT:      ## kill: YMM0<def> YMM0<kill> ZMM0<kill>
 ; KNL-NEXT:    retq
 ; SKX-LABEL: test10:
 ; SKX:       ## BB#0:
@@ -166,7 +160,6 @@ define i16 @test12(<16 x i64> %a, <16 x i64> %b) nounwind {
 ; KNL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
 ; KNL-NEXT:    kunpckbw %k0, %k1, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:      ## kill: AX<def> AX<kill> EAX<kill>
 ; KNL-NEXT:    retq
   %res = icmp eq <16 x i64> %a, %b
   %res1 = bitcast <16 x i1> %res to i16
diff --git a/test/CodeGen/X86/avx512vl-intrinsics.ll b/test/CodeGen/X86/avx512vl-intrinsics.ll
index fd76ed5d0dbdb..9d96c272f3554 100644
--- a/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -668,7 +668,7 @@ declare <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4
 
 ; CHECK-LABEL: compr7
 ; CHECK-NOT: vcompress
-; CHECK: vmovapd
+; CHECK: vmovupd
 define void @compr7(i8* %addr, <8 x double> %data) {
   call void @llvm.x86.avx512.mask.compress.store.pd.512(i8* %addr, <8 x double> %data, i8 -1)
   ret void
@@ -757,7 +757,7 @@ declare <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x
 
 ; CHECK-LABEL: expand7
 ; CHECK-NOT: vexpand
-; CHECK: vmovapd
+; CHECK: vmovupd
 define <8 x double> @expand7(i8* %addr, <8 x double> %data) {
   %res = call <8 x double> @llvm.x86.avx512.mask.expand.load.pd.512(i8* %addr, <8 x double> %data, i8 -1)
   ret <8 x double> %res
@@ -2552,4 +2552,38 @@ define <4 x float> @test_mm512_min_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %
   %res = call <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float>zeroinitializer, i8 -1)
   ret <4 x float> %res
 }
-declare <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
-\ No newline at end of file
+declare <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x double> @test_sqrt_pd_256(<4 x double> %a0, i8 %mask) {
+  ; CHECK-LABEL: test_sqrt_pd_256
+  ; CHECK: vsqrtpd
+  %res = call <4 x double> @llvm.x86.avx512.mask.sqrt.pd.256(<4 x double> %a0,  <4 x double> zeroinitializer, i8 %mask)
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx512.mask.sqrt.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+
+define <8 x float> @test_sqrt_ps_256(<8 x float> %a0, i8 %mask) {
+  ; CHECK-LABEL: test_sqrt_ps_256
+  ; CHECK: vsqrtps
+  %res = call <8 x float> @llvm.x86.avx512.mask.sqrt.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.sqrt.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+
+define <4 x double> @test_getexp_pd_256(<4 x double> %a0) {
+  ; CHECK-LABEL: test_getexp_pd_256
+  ; CHECK: vgetexppd
+  %res = call <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double> %a0,  <4 x double> zeroinitializer, i8 -1)
+  ret <4 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+
+define <8 x float> @test_getexp_ps_256(<8 x float> %a0) {
+  ; CHECK-LABEL: test_getexp_ps_256
+  ; CHECK: vgetexpps
+  %res = call <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+\ No newline at end of file
diff --git a/test/CodeGen/X86/buildvec-insertvec.ll b/test/CodeGen/X86/buildvec-insertvec.ll
index 3fb69a48b3c76..73dbe1f650a12 100644
--- a/test/CodeGen/X86/buildvec-insertvec.ll
+++ b/test/CodeGen/X86/buildvec-insertvec.ll
@@ -1,15 +1,56 @@
-; RUN: llc < %s -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s
 
 define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    cvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    movl $255, %eax
+; CHECK-NEXT:    pinsrd $3, %eax, %xmm0
+; CHECK-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    movd %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %t0 = fptoui <3 x float> %in to <3 x i8>
   %t1 = shufflevector <3 x i8> %t0, <3 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   %t2 = insertelement <4 x i8> %t1, i8 -1, i32 3
   store <4 x i8> %t2, <4 x i8>* %out, align 4
   ret void
-; CHECK: foo
-; CHECK: cvttps2dq
-; CHECK-NOT: pextrd
-; CHECK: pinsrd
-; CHECK-NEXT: pshufb
-; CHECK: ret
+}
+
+; Verify that the DAGCombiner doesn't wrongly fold a build_vector into a
+; blend with a zero vector if the build_vector contains negative zero.
+;
+; TODO: the codegen for function 'test_negative_zero_1' is sub-optimal.
+; Ideally, we should generate a single shuffle blend operation.
+
+define <4 x float> @test_negative_zero_1(<4 x float> %A) {
+; CHECK-LABEL: test_negative_zero_1:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    movapd %xmm0, %xmm1
+; CHECK-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1,0]
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    blendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    retq
+entry:
+  %0 = extractelement <4 x float> %A, i32 0
+  %1 = insertelement <4 x float> undef, float %0, i32 0
+  %2 = insertelement <4 x float> %1, float -0.0, i32 1
+  %3 = extractelement <4 x float> %A, i32 2
+  %4 = insertelement <4 x float> %2, float %3, i32 2
+  %5 = insertelement <4 x float> %4, float 0.0, i32 3
+  ret <4 x float> %5
+}
+
+define <2 x double> @test_negative_zero_2(<2 x double> %A) {
+; CHECK-LABEL: test_negative_zero_2:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    movhpd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = extractelement <2 x double> %A, i32 0
+  %1 = insertelement <2 x double> undef, double %0, i32 0
+  %2 = insertelement <2 x double> %1, double -0.0, i32 1
+  ret <2 x double> %2
 }
diff --git a/test/CodeGen/X86/critical-anti-dep-breaker.ll b/test/CodeGen/X86/critical-anti-dep-breaker.ll
index 86afc1f245ad2..de5744d3a88f7 100644
--- a/test/CodeGen/X86/critical-anti-dep-breaker.ll
+++ b/test/CodeGen/X86/critical-anti-dep-breaker.ll
@@ -11,8 +11,7 @@
 @NullToken = external global i64
 
 ; CHECK-LABEL: Part_Create:
-; CHECK-DAG: # kill: RDI<def> 
-; CHECK-DAG: movq PartClass@GOTPCREL(%rip), %r10
+; CHECK: movq PartClass@GOTPCREL(%rip), %r10
 define i32 @Part_Create(i64* %Anchor, i32 %TypeNum, i32 %F, i32 %Z, i32* %Status, i64* %PartTkn) {
   %PartObj = alloca i64*, align 8
   %Vchunk = alloca i64, align 8
diff --git a/test/CodeGen/X86/machine-cp.ll b/test/CodeGen/X86/machine-cp.ll
index 0006b6ea7133f..aaed0f0a23dca 100644
--- a/test/CodeGen/X86/machine-cp.ll
+++ b/test/CodeGen/X86/machine-cp.ll
@@ -58,3 +58,58 @@ while.end:                                        ; preds = %while.body, %entry
   %t = trunc i64 %a.addr.0.lcssa to i32
   ret i32 %t
 }
+
+; Check that copy propagation does not kill thing like:
+; dst = copy src <-- do not kill that.
+; ... = op1 dst<undef>
+; ... = op2 dst <-- this is used here.
+;
+; CHECK-LABEL: foo:
+; CHECK: psllw $7,
+; CHECK: psllw $7,
+; CHECK-NEXT: pand
+; CHECK-NEXT: pcmpgtb 
+; CHECK-NEXT: pand %xmm{{[0-9]+}}, [[SRC:%xmm[0-9]+]]
+; Machine propagation used to delete the first copy as the
+; first few uses were <undef>.
+; CHECK-NEXT: movdqa [[SRC]], [[CPY1:%xmm[0-9]+]]
+; CHECK-NEXT: movdqa [[SRC]], [[CPY2:%xmm[0-9]+]]
+; CHECK-NEXT: punpckhbw [[SRC]],
+; Check that CPY1 is not redefined.
+; CHECK-NOT: , [[CPY1]]
+; undef use, we do not care.
+; CHECK: punpcklwd [[CPY1]],
+; Check that CPY1 is not redefined.
+; CHECK-NOT: , [[CPY1]]
+; CHECK: punpcklbw [[CPY2]], [[CPY2]]
+; CHECK-NEXT: punpckhwd [[CPY2]], [[CPY2]]
+; CHECK-NEXT pslld $31, [[CPY2]]
+; Check that CPY1 is not redefined.
+; CHECK-NOT: , [[CPY1]]
+; CHECK: punpcklbw [[CPY1]], [[CPY1]]
+; CHECK-NEXT: punpcklwd [[CPY1]], [[CPY1]]
+; CHECK-NEXT pslld $31, [[CPY1]]
+define <16 x float> @foo(<16 x float> %x) {
+bb:
+  %v3 = icmp slt <16 x i32> undef, zeroinitializer
+  %v14 = zext <16 x i1> %v3 to <16 x i32>
+  %v16 = fcmp olt <16 x float> %x, zeroinitializer
+  %v17 = sext <16 x i1> %v16 to <16 x i32>
+  %v18 = zext <16 x i1> %v16 to <16 x i32>
+  %v19 = xor <16 x i32> %v14, %v18
+  %v20 = or <16 x i32> %v17, undef
+  %v21 = fptosi <16 x float> %x to <16 x i32>
+  %v22 = sitofp <16 x i32> %v21 to <16 x float>
+  %v69 = fcmp ogt <16 x float> %v22, zeroinitializer
+  %v75 = and <16 x i1> %v69, %v3
+  %v77 = bitcast <16 x float> %v22 to <16 x i32>
+  %v79 = sext <16 x i1> %v75 to <16 x i32>
+  %v80 = and <16 x i32> undef, %v79
+  %v81 = xor <16 x i32> %v77, %v80
+  %v82 = and <16 x i32> undef, %v81
+  %v83 = xor <16 x i32> %v19, %v82
+  %v84 = and <16 x i32> %v83, %v20
+  %v85 = xor <16 x i32> %v19, %v84
+  %v86 = bitcast <16 x i32> %v85 to <16 x float>
+  ret <16 x float> %v86
+}
diff --git a/test/CodeGen/X86/pic.ll b/test/CodeGen/X86/pic.ll
index d543deb804d1b..73be234db81c0 100644
--- a/test/CodeGen/X86/pic.ll
+++ b/test/CodeGen/X86/pic.ll
@@ -196,9 +196,11 @@ bb12:
 ; LINUX-NEXT: .LJTI7_0:
 ; LINUX:   .long	 .LBB7_2@GOTOFF
 ; LINUX:   .long	 .LBB7_8@GOTOFF
-; LINUX:   .long	 .LBB7_14@GOTOFF
-; LINUX:   .long	 .LBB7_9@GOTOFF
-; LINUX:   .long	 .LBB7_10@GOTOFF
+; LINUX:   .long	 .LBB7_4@GOTOFF
+; LINUX:   .long	 .LBB7_6@GOTOFF
+; LINUX:   .long	 .LBB7_5@GOTOFF
+; LINUX:   .long	 .LBB7_8@GOTOFF
+; LINUX:   .long	 .LBB7_7@GOTOFF
 }
 
 declare void @foo1(...)
diff --git a/test/CodeGen/X86/pr23603.ll b/test/CodeGen/X86/pr23603.ll
new file mode 100644
index 0000000000000..6f856aedb8d58
--- /dev/null
+++ b/test/CodeGen/X86/pr23603.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
+
+declare void @free_v()
+
+define void @f(i32* %x, i32 %c32, i32* %y) {
+; CHECK-LABEL: f
+ entry:
+  %v = load i32, i32* %x, !invariant.load !0
+; CHECK: movl (%rdi), %ebx
+; CHECK: free_v
+; CHECK-NOT: movl (%rdi), %ebx
+  call void @free_v()
+  %c = icmp ne i32 %c32, 0
+  br i1 %c, label %left, label %merge
+
+ left:
+  store i32 %v, i32* %y
+  br label %merge
+
+ merge:
+  ret void
+}
+
+!0 = !{}
diff --git a/test/CodeGen/X86/pr23664.ll b/test/CodeGen/X86/pr23664.ll
new file mode 100644
index 0000000000000..a501c0db837e3
--- /dev/null
+++ b/test/CodeGen/X86/pr23664.ll
@@ -0,0 +1,14 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i2 @f(i32 %arg) {
+  %trunc = trunc i32 %arg to i1
+  %sext = sext i1 %trunc to i2
+  %or = or i2 %sext, 1
+  ret i2 %or
+
+; CHECK-LABEL: f:
+; CHECK:      addb    %dil, %dil
+; CHECK-NEXT: orb     $1, %dil
+; CHECK-NEXT: movb    %dil, %al
+; CHECK-NEXT: retq
+}
diff --git a/test/CodeGen/X86/recip-fastmath.ll b/test/CodeGen/X86/recip-fastmath.ll
index fcd077092dab3..7f1521a83bcfd 100644
--- a/test/CodeGen/X86/recip-fastmath.ll
+++ b/test/CodeGen/X86/recip-fastmath.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,use-recip-est | FileCheck %s --check-prefix=RECIP
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,use-recip-est -x86-recip-refinement-steps=2 | FileCheck %s --check-prefix=REFINE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=divf,vec-divf | FileCheck %s --check-prefix=RECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=divf:2,vec-divf:2 | FileCheck %s --check-prefix=REFINE
 
 ; If the target's divss/divps instructions are substantially
 ; slower than rcpss/rcpps with a Newton-Raphson refinement,
diff --git a/test/CodeGen/X86/sibcall-win64.ll b/test/CodeGen/X86/sibcall-win64.ll
index f7038726f9ca5..204e1f8b050ba 100644
--- a/test/CodeGen/X86/sibcall-win64.ll
+++ b/test/CodeGen/X86/sibcall-win64.ll
@@ -1,7 +1,11 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-linux | FileCheck %s
 
 declare x86_64_win64cc void @win64_callee(i32)
+declare x86_64_win64cc void (i32)* @win64_indirect()
+declare x86_64_win64cc void @win64_other(i32)
 declare void @sysv_callee(i32)
+declare void (i32)* @sysv_indirect()
+declare void @sysv_other(i32)
 
 define void @sysv_caller(i32 %p1) {
 entry:
@@ -40,3 +44,23 @@ define x86_64_win64cc void @win64_matched(i32 %p1) {
 
 ; CHECK-LABEL: win64_matched:
 ; CHECK: jmp win64_callee # TAILCALL
+
+define x86_64_win64cc void @win64_indirect_caller(i32 %p1) {
+  %1 = call x86_64_win64cc void (i32)* @win64_indirect()
+  call x86_64_win64cc void @win64_other(i32 0)
+  tail call x86_64_win64cc void %1(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: win64_indirect_caller:
+; CHECK: jmpq *%{{rax|rcx|rdx|r8|r9|r11}} # TAILCALL
+
+define void @sysv_indirect_caller(i32 %p1) {
+  %1 = call void (i32)* @sysv_indirect()
+  call void @sysv_other(i32 0)
+  tail call void %1(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: sysv_indirect_caller:
+; CHECK: jmpq *%{{rax|rcx|rdx|rsi|rdi|r8|r9|r11}} # TAILCALL
diff --git a/test/CodeGen/X86/sqrt-fastmath.ll b/test/CodeGen/X86/sqrt-fastmath.ll
index 4c6b521156e04..373fa53c970f8 100644
--- a/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/test/CodeGen/X86/sqrt-fastmath.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,use-sqrt-est | FileCheck %s --check-prefix=ESTIMATE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=sqrtf,vec-sqrtf | FileCheck %s --check-prefix=ESTIMATE
 
 declare double @__sqrt_finite(double) #0
 declare float @__sqrtf_finite(float) #0
diff --git a/test/CodeGen/X86/stack-folding-x86_64.ll b/test/CodeGen/X86/stack-folding-x86_64.ll
new file mode 100644
index 0000000000000..211227916a09b
--- /dev/null
+++ b/test/CodeGen/X86/stack-folding-x86_64.ll
@@ -0,0 +1,51 @@
+; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; Stack reload folding tests.
+;
+; By including a nop call with sideeffects we can force a partial register spill of the
+; relevant registers and check that the reload is correctly folded into the instruction.
+
+;TODO stack_fold_bsf_i16
+declare i16 @llvm.cttz.i16(i16, i1)
+
+define i32 @stack_fold_bsf_i32(i32 %a0) {
+  ;CHECK-LABEL: stack_fold_bsf_i32
+  ;CHECK:       bsfl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i32 @llvm.cttz.i32(i32 %a0, i1 -1)
+  ret i32 %2
+}
+declare i32 @llvm.cttz.i32(i32, i1)
+
+define i64 @stack_fold_bsf_i64(i64 %a0) {
+  ;CHECK-LABEL: stack_fold_bsf_i64
+  ;CHECK:       bsfq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i64 @llvm.cttz.i64(i64 %a0, i1 -1)
+  ret i64 %2
+}
+declare i64 @llvm.cttz.i64(i64, i1)
+
+;TODO stack_fold_bsr_i16
+declare i16 @llvm.ctlz.i16(i16, i1)
+
+define i32 @stack_fold_bsr_i32(i32 %a0) {
+  ;CHECK-LABEL: stack_fold_bsr_i32
+  ;CHECK:       bsrl {{-?[0-9]*}}(%rsp), %eax {{.*#+}} 4-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i32 @llvm.ctlz.i32(i32 %a0, i1 -1)
+  ret i32 %2
+}
+declare i32 @llvm.ctlz.i32(i32, i1)
+
+define i64 @stack_fold_bsr_i64(i64 %a0) {
+  ;CHECK-LABEL: stack_fold_bsr_i64
+  ;CHECK:       bsrq {{-?[0-9]*}}(%rsp), %rax {{.*#+}} 8-byte Folded Reload
+  %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  %2 = call i64 @llvm.ctlz.i64(i64 %a0, i1 -1)
+  ret i64 %2
+}
+declare i64 @llvm.ctlz.i64(i64, i1)
diff --git a/test/CodeGen/X86/statepoint-far-call.ll b/test/CodeGen/X86/statepoint-far-call.ll
new file mode 100644
index 0000000000000..cd8dd0f35a204
--- /dev/null
+++ b/test/CodeGen/X86/statepoint-far-call.ll
@@ -0,0 +1,22 @@
+; RUN: llc < %s | FileCheck %s
+; Test to check that Statepoints with X64 far-immediate targets
+; are lowered correctly to an indirect call via a scratch register.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-win64"
+
+define void @test_far_call() gc "statepoint-example" {
+; CHECK-LABEL: test_far_call
+; CHECK: pushq %rax
+; CHECK: movabsq $140727162896504, %rax 
+; CHECK: callq *%rax
+; CHECK: popq %rax
+; CHECK: retq
+
+entry:
+  %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* inttoptr (i64 140727162896504 to void ()*), i32 0, i32 0, i32 0, i32 0)
+  ret void
+}
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) 
+
diff --git a/test/CodeGen/X86/switch-or.ll b/test/CodeGen/X86/switch-or.ll
index 6e6b013d9fa89..4642accfff8d9 100644
--- a/test/CodeGen/X86/switch-or.ll
+++ b/test/CodeGen/X86/switch-or.ll
@@ -1,10 +1,11 @@
 ; RUN: llc -march=x86 -asm-verbose=false < %s | FileCheck %s
 
 ; Check that merging switch cases that differ in one bit works.
+; CHECK-LABEL: test1
 ; CHECK: orl $2
 ; CHECK-NEXT: cmpl $6
 
-define void @foo(i32 %variable) nounwind {
+define void @test1(i32 %variable) nounwind {
 entry:
   switch i32 %variable, label %if.end [
     i32 4, label %if.then
@@ -19,4 +20,22 @@ if.end:
   ret void
 }
 
+; CHECK-LABEL: test2
+; CHECK: orl $-2147483648
+; CHECK-NEXT: cmpl $-2147483648
+define void @test2(i32 %variable) nounwind {
+entry:
+  switch i32 %variable, label %if.end [
+    i32 0, label %if.then
+    i32 -2147483648, label %if.then
+  ]
+
+if.then:
+  %call = tail call i32 (...) @bar() nounwind
+  ret void
+
+if.end:
+  ret void
+}
+
 declare i32 @bar(...) nounwind
diff --git a/test/CodeGen/X86/switch.ll b/test/CodeGen/X86/switch.ll
index 66a739c8470c4..a4dece65479c6 100644
--- a/test/CodeGen/X86/switch.ll
+++ b/test/CodeGen/X86/switch.ll
@@ -534,3 +534,18 @@ return: ret void
 ; CHECK-NOT: cmpl
 ; CHECK: cmpl $99
 }
+
+
+define void @pr23738(i4 %x) {
+entry:
+  switch i4 %x, label %bb0 [
+    i4 0, label %bb1
+    i4 1, label %bb1
+    i4 -5, label %bb1
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+; Don't assert due to truncating the bitwidth (64) to i4 when checking
+; that the bit-test range fits in a word.
+}
diff --git a/test/CodeGen/X86/tail-call-got.ll b/test/CodeGen/X86/tail-call-got.ll
index 84d561dcd8c3f..20d1a87b626a3 100644
--- a/test/CodeGen/X86/tail-call-got.ll
+++ b/test/CodeGen/X86/tail-call-got.ll
@@ -1,12 +1,14 @@
 ; RUN: llc < %s -relocation-model=pic -mattr=+sse2 | FileCheck %s
 
+; We used to do tail calls through the GOT for these symbols, but it was
+; disabled due to PR15086.
+
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 target triple = "i386-unknown-freebsd9.0"
 
 define double @test1(double %x) nounwind readnone {
 ; CHECK-LABEL: test1:
-; CHECK: movl foo@GOT
-; CHECK-NEXT: jmpl
+; CHECK:  calll foo@PLT
   %1 = tail call double @foo(double %x) nounwind readnone
   ret double %1
 }
@@ -15,10 +17,18 @@ declare double @foo(double) readnone
 
 define double @test2(double %x) nounwind readnone {
 ; CHECK-LABEL: test2:
-; CHECK: movl sin@GOT
-; CHECK-NEXT: jmpl
+; CHECK:  calll sin@PLT
   %1 = tail call double @sin(double %x) nounwind readnone
   ret double %1
 }
 
 declare double @sin(double) readnone
+
+define double @test3(double %x) nounwind readnone {
+; CHECK-LABEL: test3:
+; CHECK:  calll sin2@PLT
+  %1 = tail call double @sin2(double %x) nounwind readnone
+  ret double %1
+}
+
+declare double @sin2(double) readnone
diff --git a/test/CodeGen/X86/tailcallpic1.ll b/test/CodeGen/X86/tailcallpic1.ll
index ff590a1fd3e98..ed101fcccd2db 100644
--- a/test/CodeGen/X86/tailcallpic1.ll
+++ b/test/CodeGen/X86/tailcallpic1.ll
@@ -1,5 +1,8 @@
 ; RUN: llc < %s  -tailcallopt -mtriple=i686-pc-linux-gnu -relocation-model=pic | FileCheck %s
 
+; This test uses guaranteed TCO so these will be tail calls, despite the early
+; binding issues.
+
 define protected fastcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4) {
 entry:
 	ret i32 %a3
diff --git a/test/CodeGen/X86/tailcallpic3.ll b/test/CodeGen/X86/tailcallpic3.ll
new file mode 100644
index 0000000000000..edc58052d82f6
--- /dev/null
+++ b/test/CodeGen/X86/tailcallpic3.ll
@@ -0,0 +1,73 @@
+; RUN: llc < %s -mtriple=i686-pc-linux-gnu -relocation-model=pic | FileCheck %s
+
+; While many of these could be tail called, we don't do it because it forces
+; early binding.
+
+declare void @external()
+
+define hidden void @tailcallee_hidden() {
+entry:
+  ret void
+}
+
+define void @tailcall_hidden() {
+entry:
+  tail call void @tailcallee_hidden()
+  ret void
+}
+; CHECK: tailcall_hidden:
+; CHECK: jmp tailcallee_hidden
+
+define internal void @tailcallee_internal() {
+entry:
+  ret void
+}
+
+define void @tailcall_internal() {
+entry:
+  tail call void @tailcallee_internal()
+  ret void
+}
+; CHECK: tailcall_internal:
+; CHECK: jmp tailcallee_internal
+
+define default void @tailcallee_default() {
+entry:
+  ret void
+}
+
+define void @tailcall_default() {
+entry:
+  tail call void @tailcallee_default()
+  ret void
+}
+; CHECK: tailcall_default:
+; CHECK: calll tailcallee_default@PLT
+
+define void @tailcallee_default_implicit() {
+entry:
+  ret void
+}
+
+define void @tailcall_default_implicit() {
+entry:
+  tail call void @tailcallee_default_implicit()
+  ret void
+}
+; CHECK: tailcall_default_implicit:
+; CHECK: calll tailcallee_default_implicit@PLT
+
+define void @tailcall_external() {
+  tail call void @external()
+  ret void
+}
+; CHECK: tailcall_external:
+; CHECK: calll external@PLT
+
+define void @musttail_external() {
+  musttail call void @external()
+  ret void
+}
+; CHECK: musttail_external:
+; CHECK: movl external@GOT
+; CHECK: jmpl
diff --git a/test/CodeGen/X86/vec_fp_to_int.ll b/test/CodeGen/X86/vec_fp_to_int.ll
index 9f1c7afa295b5..3e72212d85d3e 100644
--- a/test/CodeGen/X86/vec_fp_to_int.ll
+++ b/test/CodeGen/X86/vec_fp_to_int.ll
@@ -239,7 +239,6 @@ define <4 x i64> @fptoui_4vf64(<4 x double> %a) {
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    movapd %xmm0, %xmm2
 ; SSE2-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
-; SSE2-NEXT:    {{.*#+}} kill: XMM0<def> XMM2<kill>
 ; SSE2-NEXT:    subsd %xmm3, %xmm0
 ; SSE2-NEXT:    cvttsd2si %xmm0, %rcx
 ; SSE2-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
@@ -589,7 +588,6 @@ define <8 x i32> @fptoui_8vf32(<8 x float> %a) {
 ; SSE2-LABEL: fptoui_8vf32:
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    {{.*#+}} kill: XMM0<def> XMM2<kill>
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    cvttss2si %xmm0, %rax
 ; SSE2-NEXT:    movd %eax, %xmm0
diff --git a/test/CodeGen/X86/vec_shift8.ll b/test/CodeGen/X86/vec_shift8.ll
new file mode 100644
index 0000000000000..a32cb30b0b262
--- /dev/null
+++ b/test/CodeGen/X86/vec_shift8.ll
@@ -0,0 +1,1016 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX
+
+;
+; Vectorized integer shifts
+;
+
+define <2 x i64> @shl_8i16(<8 x i16> %r, <8 x i16> %a) nounwind readnone ssp {
+entry:
+; SSE2:       pextrw $7, %xmm0, %eax
+; SSE2-NEXT:  pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm2
+; SSE2-NEXT:  pextrw $3, %xmm0, %eax
+; SSE2-NEXT:  pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:  pextrw $5, %xmm0, %eax
+; SSE2-NEXT:  pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm4
+; SSE2-NEXT:  pextrw $1, %xmm0, %eax
+; SSE2-NEXT:  pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm2
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; SSE2-NEXT:  pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  pextrw $2, %xmm0, %eax
+; SSE2-NEXT:  pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm4
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:  pextrw $4, %xmm0, %eax
+; SSE2-NEXT:  pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  movd %xmm0, %eax
+; SSE2-NEXT:  movd %xmm1, %ecx
+; SSE2-NEXT:  shll %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:  retq
+;
+; SSE41:      pextrw $1, %xmm0, %eax
+; SSE41-NEXT: pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: movd %xmm0, %edx
+; SSE41-NEXT: movd %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %edx
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrw $1, %eax, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: pinsrw $2, %eax, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: pinsrw $3, %eax, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: pinsrw $5, %eax, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: pinsrw $6, %eax, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: shll %cl, %eax
+; SSE41-NEXT: pinsrw $7, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX:        vpextrw $1, %xmm0, %eax
+; AVX-NEXT:   vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vmovd %xmm0, %edx
+; AVX-NEXT:   vmovd %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %edx
+; AVX-NEXT:   vmovd %edx, %xmm2
+; AVX-NEXT:   vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $2, %xmm0, %eax
+; AVX-NEXT:   vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $3, %xmm0, %eax
+; AVX-NEXT:   vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $4, %xmm0, %eax
+; AVX-NEXT:   vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $5, %xmm0, %eax
+; AVX-NEXT:   vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $6, %xmm0, %eax
+; AVX-NEXT:   vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $7, %xmm0, %eax
+; AVX-NEXT:   vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:   shll %cl, %eax
+; AVX-NEXT:   vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT:   retq
+  %shl = shl <8 x i16> %r, %a
+  %tmp2 = bitcast <8 x i16> %shl to <2 x i64>
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @shl_16i8(<16 x i8> %r, <16 x i8> %a) nounwind readnone ssp {
+entry:
+; SSE2:       psllw $5, %xmm1
+; SSE2-NEXT:  pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:  movdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; SSE2-NEXT:  movdqa %xmm2, %xmm3
+; SSE2-NEXT:  pand %xmm1, %xmm3
+; SSE2-NEXT:  pcmpeqb %xmm2, %xmm3
+; SSE2-NEXT:  movdqa %xmm3, %xmm4
+; SSE2-NEXT:  pandn %xmm0, %xmm4
+; SSE2-NEXT:  psllw $4, %xmm0
+; SSE2-NEXT:  pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:  pand %xmm3, %xmm0
+; SSE2-NEXT:  por %xmm4, %xmm0
+; SSE2-NEXT:  paddb %xmm1, %xmm1
+; SSE2-NEXT:  movdqa %xmm2, %xmm3
+; SSE2-NEXT:  pand %xmm1, %xmm3
+; SSE2-NEXT:  pcmpeqb %xmm2, %xmm3
+; SSE2-NEXT:  movdqa %xmm3, %xmm4
+; SSE2-NEXT:  pandn %xmm0, %xmm4
+; SSE2-NEXT:  psllw $2, %xmm0
+; SSE2-NEXT:  pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:  pand %xmm3, %xmm0
+; SSE2-NEXT:  por %xmm4, %xmm0
+; SSE2-NEXT:  paddb %xmm1, %xmm1
+; SSE2-NEXT:  pand %xmm2, %xmm1
+; SSE2-NEXT:  pcmpeqb %xmm2, %xmm1
+; SSE2-NEXT:  movdqa %xmm1, %xmm2
+; SSE2-NEXT:  pandn %xmm0, %xmm2
+; SSE2-NEXT:  paddb %xmm0, %xmm0
+; SSE2-NEXT:  pand %xmm1, %xmm0
+; SSE2-NEXT:  por %xmm2, %xmm0
+; SSE2-NEXT:  retq
+;
+; SSE41:      movdqa %xmm0, %xmm2
+; SSE41-NEXT: psllw $5, %xmm1
+; SSE41-NEXT: pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT: movdqa %xmm1, %xmm5
+; SSE41-NEXT: paddb %xmm5, %xmm5
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; SSE41-NEXT: movdqa %xmm3, %xmm4
+; SSE41-NEXT: pand %xmm5, %xmm4
+; SSE41-NEXT: pcmpeqb %xmm3, %xmm4
+; SSE41-NEXT: pand %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqb %xmm3, %xmm1
+; SSE41-NEXT: movdqa %xmm2, %xmm6
+; SSE41-NEXT: psllw $4, %xmm6
+; SSE41-NEXT: pand {{.*}}(%rip), %xmm6
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pblendvb %xmm6, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm1
+; SSE41-NEXT: psllw $2, %xmm1
+; SSE41-NEXT: pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT: movdqa %xmm4, %xmm0
+; SSE41-NEXT: pblendvb %xmm1, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm1
+; SSE41-NEXT: paddb %xmm1, %xmm1
+; SSE41-NEXT: paddb %xmm5, %xmm5
+; SSE41-NEXT: pand %xmm3, %xmm5
+; SSE41-NEXT: pcmpeqb %xmm5, %xmm3
+; SSE41-NEXT: movdqa %xmm3, %xmm0
+; SSE41-NEXT: pblendvb %xmm1, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX:        vpsllw $5, %xmm1, %xmm1
+; AVX-NEXT:   vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:   vpaddb %xmm1, %xmm1, %xmm2
+; AVX-NEXT:   vmovdqa {{.*#+}} xmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX-NEXT:   vpand %xmm2, %xmm3, %xmm4
+; AVX-NEXT:   vpcmpeqb %xmm3, %xmm4, %xmm4
+; AVX-NEXT:   vpand %xmm1, %xmm3, %xmm1
+; AVX-NEXT:   vpcmpeqb %xmm3, %xmm1, %xmm1
+; AVX-NEXT:   vpsllw $4, %xmm0, %xmm5
+; AVX-NEXT:   vpand {{.*}}(%rip), %xmm5, %xmm5
+; AVX-NEXT:   vpblendvb %xmm1, %xmm5, %xmm0, %xmm0
+; AVX-NEXT:   vpsllw $2, %xmm0, %xmm1
+; AVX-NEXT:   vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:   vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:   vpaddb %xmm0, %xmm0, %xmm1
+; AVX-NEXT:   vpaddb %xmm2, %xmm2, %xmm2
+; AVX-NEXT:   vpand %xmm2, %xmm3, %xmm2
+; AVX-NEXT:   vpcmpeqb %xmm3, %xmm2, %xmm2
+; AVX-NEXT:   vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:   retq
+  %shl = shl <16 x i8> %r, %a
+  %tmp2 = bitcast <16 x i8> %shl to <2 x i64>
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @ashr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind readnone ssp {
+entry:
+; SSE2:       pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $7, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm2
+; SSE2-NEXT:  pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $3, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:  pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $5, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm4
+; SSE2-NEXT:  pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $1, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm2
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; SSE2-NEXT:  pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $2, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm4
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:  pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:  pextrw $4, %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  movd %xmm1, %ecx
+; SSE2-NEXT:  movd %xmm0, %eax
+; SSE2-NEXT:  sarw %cl, %ax
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:  retq
+;
+; SSE41:      pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: pextrw $1, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: movd %xmm1, %ecx
+; SSE41-NEXT: movd %xmm0, %edx
+; SSE41-NEXT: sarw %cl, %dx
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrw $1, %eax, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: pinsrw $2, %eax, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: pinsrw $3, %eax, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: pinsrw $5, %eax, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: pinsrw $6, %eax, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: sarw %cl, %ax
+; SSE41-NEXT: pinsrw $7, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX:        vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $1, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vmovd %xmm1, %ecx
+; AVX-NEXT:   vmovd %xmm0, %edx
+; AVX-NEXT:   sarw %cl, %dx
+; AVX-NEXT:   vmovd %edx, %xmm2
+; AVX-NEXT:   vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $2, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $3, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $4, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $5, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $6, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:   vpextrw $7, %xmm0, %eax
+; AVX-NEXT:   sarw %cl, %ax
+; AVX-NEXT:   vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT:   retq
+  %ashr = ashr <8 x i16> %r, %a
+  %tmp2 = bitcast <8 x i16> %ashr to <2 x i64>
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @ashr_16i8(<16 x i8> %r, <16 x i8> %a) nounwind readnone ssp {
+entry:
+;
+; SSE2:       pushq %rbp
+; SSE2-NEXT:  pushq %r15
+; SSE2-NEXT:  pushq %r14
+; SSE2-NEXT:  pushq %r13
+; SSE2-NEXT:  pushq %r12
+; SSE2-NEXT:  pushq %rbx
+; SSE2-NEXT:  movaps %xmm1, -24(%rsp)
+; SSE2-NEXT:  movaps %xmm0, -40(%rsp)
+; SSE2-NEXT:  movb -9(%rsp), %cl
+; SSE2-NEXT:  movb -25(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movzbl %al, %eax
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  movb -17(%rsp), %cl
+; SSE2-NEXT:  movb -33(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -13(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %eax
+; SSE2-NEXT:  movl %eax, -44(%rsp)
+; SSE2-NEXT:  movb -29(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movzbl %al, %r9d
+; SSE2-NEXT:  movb -21(%rsp), %cl
+; SSE2-NEXT:  movb -37(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -11(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r10d
+; SSE2-NEXT:  movb -27(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -19(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r11d
+; SSE2-NEXT:  movb -35(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -15(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r14d
+; SSE2-NEXT:  movb -31(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movzbl %al, %r15d
+; SSE2-NEXT:  movb -23(%rsp), %cl
+; SSE2-NEXT:  movb -39(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -10(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r12d
+; SSE2-NEXT:  movb -26(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -18(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r13d
+; SSE2-NEXT:  movb -34(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -14(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r8d
+; SSE2-NEXT:  movb -30(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -22(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %ebp
+; SSE2-NEXT:  movb -38(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movb -12(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %edi
+; SSE2-NEXT:  movb -28(%rsp), %dl
+; SSE2-NEXT:  sarb %cl, %dl
+; SSE2-NEXT:  movb -20(%rsp), %cl
+; SSE2-NEXT:  movzbl %dl, %esi
+; SSE2-NEXT:  movb -36(%rsp), %bl
+; SSE2-NEXT:  sarb %cl, %bl
+; SSE2-NEXT:  movb -16(%rsp), %cl
+; SSE2-NEXT:  movzbl %bl, %ebx
+; SSE2-NEXT:  movb -32(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movzbl %al, %edx
+; SSE2-NEXT:  movb -24(%rsp), %cl
+; SSE2-NEXT:  movb -40(%rsp), %al
+; SSE2-NEXT:  sarb %cl, %al
+; SSE2-NEXT:  movzbl %al, %eax
+; SSE2-NEXT:  movd -44(%rsp), %xmm1
+; SSE2:       movd %r9d, %xmm2
+; SSE2-NEXT:  movd %r10d, %xmm3
+; SSE2-NEXT:  movd %r11d, %xmm4
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:  movd %r14d, %xmm0
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:  movd %r15d, %xmm1
+; SSE2-NEXT:  movd %r12d, %xmm2
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:  movd %r13d, %xmm0
+; SSE2-NEXT:  movd %r8d, %xmm1
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:  movd %ebp, %xmm0
+; SSE2-NEXT:  movd %edi, %xmm3
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:  movd %esi, %xmm0
+; SSE2-NEXT:  movd %ebx, %xmm1
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:  movd %edx, %xmm4
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:  popq %rbx
+; SSE2-NEXT:  popq %r12
+; SSE2-NEXT:  popq %r13
+; SSE2-NEXT:  popq %r14
+; SSE2-NEXT:  popq %r15
+; SSE2-NEXT:  popq %rbp
+; SSE2-NEXT:  retq
+;
+; SSE41:      pextrb $1, %xmm1, %ecx
+; SSE41-NEXT: pextrb $1, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pextrb $0, %xmm1, %ecx
+; SSE41-NEXT: pextrb $0, %xmm0, %edx
+; SSE41-NEXT: sarb %cl, %dl
+; SSE41-NEXT: movzbl %dl, %ecx
+; SSE41-NEXT: movd %ecx, %xmm2
+; SSE41-NEXT: pinsrb $1, %eax, %xmm2
+; SSE41-NEXT: pextrb $2, %xmm1, %ecx
+; SSE41-NEXT: pextrb $2, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $2, %eax, %xmm2
+; SSE41-NEXT: pextrb $3, %xmm1, %ecx
+; SSE41-NEXT: pextrb $3, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $3, %eax, %xmm2
+; SSE41-NEXT: pextrb $4, %xmm1, %ecx
+; SSE41-NEXT: pextrb $4, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $4, %eax, %xmm2
+; SSE41-NEXT: pextrb $5, %xmm1, %ecx
+; SSE41-NEXT: pextrb $5, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $5, %eax, %xmm2
+; SSE41-NEXT: pextrb $6, %xmm1, %ecx
+; SSE41-NEXT: pextrb $6, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $6, %eax, %xmm2
+; SSE41-NEXT: pextrb $7, %xmm1, %ecx
+; SSE41-NEXT: pextrb $7, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $7, %eax, %xmm2
+; SSE41-NEXT: pextrb $8, %xmm1, %ecx
+; SSE41-NEXT: pextrb $8, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $8, %eax, %xmm2
+; SSE41-NEXT: pextrb $9, %xmm1, %ecx
+; SSE41-NEXT: pextrb $9, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $9, %eax, %xmm2
+; SSE41-NEXT: pextrb $10, %xmm1, %ecx
+; SSE41-NEXT: pextrb $10, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $10, %eax, %xmm2
+; SSE41-NEXT: pextrb $11, %xmm1, %ecx
+; SSE41-NEXT: pextrb $11, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $11, %eax, %xmm2
+; SSE41-NEXT: pextrb $12, %xmm1, %ecx
+; SSE41-NEXT: pextrb $12, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $12, %eax, %xmm2
+; SSE41-NEXT: pextrb $13, %xmm1, %ecx
+; SSE41-NEXT: pextrb $13, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $13, %eax, %xmm2
+; SSE41-NEXT: pextrb $14, %xmm1, %ecx
+; SSE41-NEXT: pextrb $14, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $14, %eax, %xmm2
+; SSE41-NEXT: pextrb $15, %xmm1, %ecx
+; SSE41-NEXT: pextrb $15, %xmm0, %eax
+; SSE41-NEXT: sarb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $15, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX:        vpextrb $1, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $1, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpextrb $0, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $0, %xmm0, %edx
+; AVX-NEXT:   sarb %cl, %dl
+; AVX-NEXT:   movzbl %dl, %ecx
+; AVX-NEXT:   vmovd %ecx, %xmm2
+; AVX-NEXT:   vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $2, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $2, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $3, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $3, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $4, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $4, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $5, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $5, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $6, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $6, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $7, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $7, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $8, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $8, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $9, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $9, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $10, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $10, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $11, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $11, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $12, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $12, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $13, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $13, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $14, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $14, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $15, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $15, %xmm0, %eax
+; AVX-NEXT:   sarb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX-NEXT:   retq
+  %ashr = ashr <16 x i8> %r, %a
+  %tmp2 = bitcast <16 x i8> %ashr to <2 x i64>
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @lshr_8i16(<8 x i16> %r, <8 x i16> %a) nounwind readnone ssp {
+entry:
+
+; SSE2:       pextrw $7, %xmm0, %eax
+; SSE2-NEXT:  pextrw $7, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm2
+; SSE2-NEXT:  pextrw $3, %xmm0, %eax
+; SSE2-NEXT:  pextrw $3, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:  pextrw $5, %xmm0, %eax
+; SSE2-NEXT:  pextrw $5, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm4
+; SSE2-NEXT:  pextrw $1, %xmm0, %eax
+; SSE2-NEXT:  pextrw $1, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm2
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; SSE2-NEXT:  pextrw $6, %xmm0, %eax
+; SSE2-NEXT:  pextrw $6, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  pextrw $2, %xmm0, %eax
+; SSE2-NEXT:  pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm4
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; SSE2-NEXT:  pextrw $4, %xmm0, %eax
+; SSE2-NEXT:  pextrw $4, %xmm1, %ecx
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm3
+; SSE2-NEXT:  movd %xmm1, %ecx
+; SSE2-NEXT:  movd %xmm0, %eax
+; SSE2-NEXT:  movzwl %ax, %eax
+; SSE2-NEXT:  shrl %cl, %eax
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT:  punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:  retq
+;
+; SSE41:      pextrw $1, %xmm0, %eax
+; SSE41-NEXT: pextrw $1, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: movd %xmm1, %ecx
+; SSE41-NEXT: movd %xmm0, %edx
+; SSE41-NEXT: movzwl %dx, %edx
+; SSE41-NEXT: shrl %cl, %edx
+; SSE41-NEXT: movd %edx, %xmm2
+; SSE41-NEXT: pinsrw $1, %eax, %xmm2
+; SSE41-NEXT: pextrw $2, %xmm0, %eax
+; SSE41-NEXT: pextrw $2, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: pinsrw $2, %eax, %xmm2
+; SSE41-NEXT: pextrw $3, %xmm0, %eax
+; SSE41-NEXT: pextrw $3, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: pinsrw $3, %eax, %xmm2
+; SSE41-NEXT: pextrw $4, %xmm0, %eax
+; SSE41-NEXT: pextrw $4, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: pinsrw $4, %eax, %xmm2
+; SSE41-NEXT: pextrw $5, %xmm0, %eax
+; SSE41-NEXT: pextrw $5, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: pinsrw $5, %eax, %xmm2
+; SSE41-NEXT: pextrw $6, %xmm0, %eax
+; SSE41-NEXT: pextrw $6, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: pinsrw $6, %eax, %xmm2
+; SSE41-NEXT: pextrw $7, %xmm0, %eax
+; SSE41-NEXT: pextrw $7, %xmm1, %ecx
+; SSE41-NEXT: shrl %cl, %eax
+; SSE41-NEXT: pinsrw $7, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX:        vpextrw $1, %xmm0, %eax
+; AVX-NEXT:   vpextrw $1, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vmovd %xmm1, %ecx
+; AVX-NEXT:   vmovd %xmm0, %edx
+; AVX-NEXT:   movzwl %dx, %edx
+; AVX-NEXT:   shrl %cl, %edx
+; AVX-NEXT:   vmovd %edx, %xmm2
+; AVX-NEXT:   vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $2, %xmm0, %eax
+; AVX-NEXT:   vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $3, %xmm0, %eax
+; AVX-NEXT:   vpextrw $3, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $4, %xmm0, %eax
+; AVX-NEXT:   vpextrw $4, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $5, %xmm0, %eax
+; AVX-NEXT:   vpextrw $5, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $6, %xmm0, %eax
+; AVX-NEXT:   vpextrw $6, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrw $7, %xmm0, %eax
+; AVX-NEXT:   vpextrw $7, %xmm1, %ecx
+; AVX-NEXT:   shrl %cl, %eax
+; AVX-NEXT:   vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX-NEXT:   retq
+  %lshr = lshr <8 x i16> %r, %a
+  %tmp2 = bitcast <8 x i16> %lshr to <2 x i64>
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @lshr_16i8(<16 x i8> %r, <16 x i8> %a) nounwind readnone ssp {
+entry:
+; SSE2:       pushq %rbp
+; SSE2-NEXT:  pushq %r15
+; SSE2-NEXT:  pushq %r14
+; SSE2-NEXT:  pushq %r13
+; SSE2-NEXT:  pushq %r12
+; SSE2-NEXT:  pushq %rbx
+; SSE2-NEXT:  movaps %xmm1, -24(%rsp)
+; SSE2-NEXT:  movaps %xmm0, -40(%rsp)
+; SSE2-NEXT:  movb -9(%rsp), %cl
+; SSE2-NEXT:  movb -25(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movzbl %al, %eax
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  movb -17(%rsp), %cl
+; SSE2-NEXT:  movb -33(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -13(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %eax
+; SSE2-NEXT:  movl %eax, -44(%rsp)
+; SSE2-NEXT:  movb -29(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movzbl %al, %r9d
+; SSE2-NEXT:  movb -21(%rsp), %cl
+; SSE2-NEXT:  movb -37(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -11(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r10d
+; SSE2-NEXT:  movb -27(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -19(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r11d
+; SSE2-NEXT:  movb -35(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -15(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r14d
+; SSE2-NEXT:  movb -31(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movzbl %al, %r15d
+; SSE2-NEXT:  movb -23(%rsp), %cl
+; SSE2-NEXT:  movb -39(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -10(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r12d
+; SSE2-NEXT:  movb -26(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -18(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r13d
+; SSE2-NEXT:  movb -34(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -14(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %r8d
+; SSE2-NEXT:  movb -30(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -22(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %ebp
+; SSE2-NEXT:  movb -38(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movb -12(%rsp), %cl
+; SSE2-NEXT:  movzbl %al, %edi
+; SSE2-NEXT:  movb -28(%rsp), %dl
+; SSE2-NEXT:  shrb %cl, %dl
+; SSE2-NEXT:  movb -20(%rsp), %cl
+; SSE2-NEXT:  movzbl %dl, %esi
+; SSE2-NEXT:  movb -36(%rsp), %bl
+; SSE2-NEXT:  shrb %cl, %bl
+; SSE2-NEXT:  movb -16(%rsp), %cl
+; SSE2-NEXT:  movzbl %bl, %ebx
+; SSE2-NEXT:  movb -32(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movzbl %al, %edx
+; SSE2-NEXT:  movb -24(%rsp), %cl
+; SSE2-NEXT:  movb -40(%rsp), %al
+; SSE2-NEXT:  shrb %cl, %al
+; SSE2-NEXT:  movzbl %al, %eax
+; SSE2-NEXT:  movd -44(%rsp), %xmm1
+; SSE2:       movd %r9d, %xmm2
+; SSE2-NEXT:  movd %r10d, %xmm3
+; SSE2-NEXT:  movd %r11d, %xmm4
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:  movd %r14d, %xmm0
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:  movd %r15d, %xmm1
+; SSE2-NEXT:  movd %r12d, %xmm2
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:  movd %r13d, %xmm0
+; SSE2-NEXT:  movd %r8d, %xmm1
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:  movd %ebp, %xmm0
+; SSE2-NEXT:  movd %edi, %xmm3
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:  movd %esi, %xmm0
+; SSE2-NEXT:  movd %ebx, %xmm1
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:  movd %edx, %xmm4
+; SSE2-NEXT:  movd %eax, %xmm0
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:  punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:  popq %rbx
+; SSE2-NEXT:  popq %r12
+; SSE2-NEXT:  popq %r13
+; SSE2-NEXT:  popq %r14
+; SSE2-NEXT:  popq %r15
+; SSE2-NEXT:  popq %rbp
+; SSE2-NEXT:  retq
+;
+; SSE41:      pextrb $1, %xmm1, %ecx
+; SSE41-NEXT: pextrb $1, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pextrb $0, %xmm1, %ecx
+; SSE41-NEXT: pextrb $0, %xmm0, %edx
+; SSE41-NEXT: shrb %cl, %dl
+; SSE41-NEXT: movzbl %dl, %ecx
+; SSE41-NEXT: movd %ecx, %xmm2
+; SSE41-NEXT: pinsrb $1, %eax, %xmm2
+; SSE41-NEXT: pextrb $2, %xmm1, %ecx
+; SSE41-NEXT: pextrb $2, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $2, %eax, %xmm2
+; SSE41-NEXT: pextrb $3, %xmm1, %ecx
+; SSE41-NEXT: pextrb $3, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $3, %eax, %xmm2
+; SSE41-NEXT: pextrb $4, %xmm1, %ecx
+; SSE41-NEXT: pextrb $4, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $4, %eax, %xmm2
+; SSE41-NEXT: pextrb $5, %xmm1, %ecx
+; SSE41-NEXT: pextrb $5, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $5, %eax, %xmm2
+; SSE41-NEXT: pextrb $6, %xmm1, %ecx
+; SSE41-NEXT: pextrb $6, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $6, %eax, %xmm2
+; SSE41-NEXT: pextrb $7, %xmm1, %ecx
+; SSE41-NEXT: pextrb $7, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $7, %eax, %xmm2
+; SSE41-NEXT: pextrb $8, %xmm1, %ecx
+; SSE41-NEXT: pextrb $8, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $8, %eax, %xmm2
+; SSE41-NEXT: pextrb $9, %xmm1, %ecx
+; SSE41-NEXT: pextrb $9, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $9, %eax, %xmm2
+; SSE41-NEXT: pextrb $10, %xmm1, %ecx
+; SSE41-NEXT: pextrb $10, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $10, %eax, %xmm2
+; SSE41-NEXT: pextrb $11, %xmm1, %ecx
+; SSE41-NEXT: pextrb $11, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $11, %eax, %xmm2
+; SSE41-NEXT: pextrb $12, %xmm1, %ecx
+; SSE41-NEXT: pextrb $12, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $12, %eax, %xmm2
+; SSE41-NEXT: pextrb $13, %xmm1, %ecx
+; SSE41-NEXT: pextrb $13, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $13, %eax, %xmm2
+; SSE41-NEXT: pextrb $14, %xmm1, %ecx
+; SSE41-NEXT: pextrb $14, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $14, %eax, %xmm2
+; SSE41-NEXT: pextrb $15, %xmm1, %ecx
+; SSE41-NEXT: pextrb $15, %xmm0, %eax
+; SSE41-NEXT: shrb %cl, %al
+; SSE41-NEXT: movzbl %al, %eax
+; SSE41-NEXT: pinsrb $15, %eax, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX:        vpextrb $1, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $1, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpextrb $0, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $0, %xmm0, %edx
+; AVX-NEXT:   shrb %cl, %dl
+; AVX-NEXT:   movzbl %dl, %ecx
+; AVX-NEXT:   vmovd %ecx, %xmm2
+; AVX-NEXT:   vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $2, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $2, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $3, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $3, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $4, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $4, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $5, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $5, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $6, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $6, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $7, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $7, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $8, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $8, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $9, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $9, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $10, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $10, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $11, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $11, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $12, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $12, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $13, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $13, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $14, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $14, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX-NEXT:   vpextrb $15, %xmm1, %ecx
+; AVX-NEXT:   vpextrb $15, %xmm0, %eax
+; AVX-NEXT:   shrb %cl, %al
+; AVX-NEXT:   movzbl %al, %eax
+; AVX-NEXT:   vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX-NEXT:   retq
+  %lshr = lshr <16 x i8> %r, %a
+  %tmp2 = bitcast <16 x i8> %lshr to <2 x i64>
+  ret <2 x i64> %tmp2
+}
diff --git a/test/CodeGen/X86/vector-ctpop.ll b/test/CodeGen/X86/vector-ctpop.ll
deleted file mode 100644
index 59d67928c6fad..0000000000000
--- a/test/CodeGen/X86/vector-ctpop.ll
+++ /dev/null
@@ -1,159 +0,0 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx2                | FileCheck -check-prefix=AVX2 %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx  -mattr=-popcnt | FileCheck -check-prefix=AVX1-NOPOPCNT %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx2 -mattr=-popcnt | FileCheck -check-prefix=AVX2-NOPOPCNT %s
-
-; Vector version of:
-; v = v - ((v >> 1) & 0x55555555)
-; v = (v & 0x33333333) + ((v >> 2) & 0x33333333)
-; v = (v + (v >> 4) & 0xF0F0F0F)
-; v = v + (v >> 8)
-; v = v + (v >> 16)
-; v = v + (v >> 32) ; i64 only
-
-define <8 x i32> @test0(<8 x i32> %x) {
-; AVX2-LABEL: @test0
-entry:
-; AVX2:  vpsrld  $1, %ymm
-; AVX2-NEXT:  vpbroadcastd
-; AVX2-NEXT:  vpand
-; AVX2-NEXT:  vpsubd
-; AVX2-NEXT:  vpbroadcastd
-; AVX2-NEXT:  vpand
-; AVX2-NEXT:  vpsrld  $2
-; AVX2-NEXT:  vpand
-; AVX2-NEXT:  vpaddd
-; AVX2-NEXT:  vpsrld  $4
-; AVX2-NEXT:  vpaddd
-; AVX2-NEXT:  vpbroadcastd
-; AVX2-NEXT:	vpand
-; AVX2-NEXT:	vpsrld	$8
-; AVX2-NEXT:	vpaddd
-; AVX2-NEXT:	vpsrld	$16
-; AVX2-NEXT:	vpaddd
-; AVX2-NEXT:	vpbroadcastd
-; AVX2-NEXT:	vpand
-  %y = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %x)
-  ret <8 x i32> %y
-}
-
-define <4 x i64> @test1(<4 x i64> %x) {
-; AVX2-NOPOPCNT-LABEL: @test1
-entry:
-;	AVX2-NOPOPCNT: vpsrlq	$1, %ymm
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpsubq
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$2
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$4
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$8
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$16
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpsrlq	$32
-;	AVX2-NOPOPCNT-NEXT: vpaddq
-;	AVX2-NOPOPCNT-NEXT: vpbroadcastq
-;	AVX2-NOPOPCNT-NEXT: vpand
-  %y = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %x)
-  ret <4 x i64> %y
-}
-
-define <4 x i32> @test2(<4 x i32> %x) {
-; AVX2-NOPOPCNT-LABEL: @test2
-; AVX1-NOPOPCNT-LABEL: @test2
-entry:
-; AVX2-NOPOPCNT:	vpsrld	$1, %xmm
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsubd
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$2
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$4
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$8
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpsrld	$16
-; AVX2-NOPOPCNT-NEXT:	vpaddd
-; AVX2-NOPOPCNT-NEXT:	vpbroadcastd
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT:	vpsrld	$1, %xmm
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsubd
-; AVX1-NOPOPCNT-NEXT:	vmovdqa
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$2
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$4
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$8
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpsrld	$16
-; AVX1-NOPOPCNT-NEXT:	vpaddd
-; AVX1-NOPOPCNT-NEXT:	vpand
-  %y = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %x)
-  ret <4 x i32> %y
-}
-
-define <2 x i64> @test3(<2 x i64> %x) {
-; AVX2-NOPOPCNT-LABEL: @test3
-; AVX1-NOPOPCNT-LABEL: @test3
-entry:
-; AVX2-NOPOPCNT:	vpsrlq	$1, %xmm
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsubq
-; AVX2-NOPOPCNT-NEXT:	vmovdqa
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$2
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$4
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$8
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$16
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpsrlq	$32
-; AVX2-NOPOPCNT-NEXT:	vpaddq
-; AVX2-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT:	vpsrlq	$1, %xmm
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsubq
-; AVX1-NOPOPCNT-NEXT:	vmovdqa
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$2
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$4
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpand
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$8
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$16
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpsrlq	$32
-; AVX1-NOPOPCNT-NEXT:	vpaddq
-; AVX1-NOPOPCNT-NEXT:	vpand
-  %y = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %x)
-  ret <2 x i64> %y
-}
-
-declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
-declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
-
-declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
-declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
-
diff --git a/test/CodeGen/X86/vector-lzcnt-128.ll b/test/CodeGen/X86/vector-lzcnt-128.ll
new file mode 100644
index 0000000000000..b43188b7c6ea1
--- /dev/null
+++ b/test/CodeGen/X86/vector-lzcnt-128.ll
@@ -0,0 +1,1915 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <2 x i64> @testv2i64(<2 x i64> %in) {
+; SSE2-LABEL: testv2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    movl $127, %ecx
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd $78, %xmm0, %xmm0       # xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq %xmm0, %xmm1    # xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    movl $127, %ecx
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd $78, %xmm0, %xmm0       # xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq %xmm0, %xmm1    # xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    movl $127, %ecx
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd $78, %xmm0, %xmm0       # xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq %xmm0, %xmm1    # xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+
+;
+; SSE41-LABEL: testv2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    movl $127, %ecx
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    movl $127, %ecx
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @testv2i64u(<2 x i64> %in) {
+; SSE2-LABEL: testv2i64u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsrq %rax, %rax
+; SSE2-NEXT:    xorq $63, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsrq %rax, %rax
+; SSE3-NEXT:    xorq $63, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsrq %rax, %rax
+; SSSE3-NEXT:    xorq $63, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsrq %rax, %rax
+; SSE41-NEXT:    xorq $63, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsrq %rax, %rax
+; AVX-NEXT:    xorq $63, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @testv4i32(<4 x i32> %in) {
+; SSE2-LABEL: testv4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    movl $63, %ecx
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    movl $63, %ecx
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    movl $63, %ecx
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    movl $63, %ecx
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsrl %edx, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    xorl $31, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    movl $63, %ecx
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsrl %edx, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    xorl $31, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %in, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @testv4i32u(<4 x i32> %in) {
+; SSE2-LABEL: testv4i32u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $31, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $31, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    xorl $31, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $31, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    xorl $31, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $31, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %in, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @testv8i16(<8 x i16> %in) {
+; SSE2-LABEL: testv8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %cx
+; SSE2-NEXT:    movw $31, %ax
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    bsrw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    xorl $15, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %cx
+; SSE3-NEXT:    movw $31, %ax
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    bsrw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    xorl $15, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %cx
+; SSSE3-NEXT:    movw $31, %ax
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movd %xmm0, %ecx
+; SSSE3-NEXT:    bsrw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    xorl $15, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %cx
+; SSE41-NEXT:    movw $31, %ax
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsrw %dx, %dx
+; SSE41-NEXT:    cmovew %ax, %dx
+; SSE41-NEXT:    xorl $15, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    pinsrw $7, %ecx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %cx
+; AVX-NEXT:    movw $31, %ax
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsrw %dx, %dx
+; AVX-NEXT:    cmovew %ax, %dx
+; AVX-NEXT:    xorl $15, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vpinsrw $7, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %in, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @testv8i16u(<8 x i16> %in) {
+; SSE2-LABEL: testv8i16u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsrw %ax, %ax
+; SSE2-NEXT:    xorl $15, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsrw %ax, %ax
+; SSE3-NEXT:    xorl $15, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsrw %ax, %ax
+; SSSE3-NEXT:    xorl $15, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsrw %cx, %cx
+; SSE41-NEXT:    xorl $15, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %eax, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $2, %eax, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $3, %eax, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $4, %eax, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $5, %eax, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $6, %eax, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %eax
+; SSE41-NEXT:    bsrw %ax, %ax
+; SSE41-NEXT:    xorl $15, %eax
+; SSE41-NEXT:    pinsrw $7, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsrw %cx, %cx
+; AVX-NEXT:    xorl $15, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    bsrw %ax, %ax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %in, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @testv16i8(<16 x i8> %in) {
+; SSE2-LABEL: testv16i8:
+; SSE2:       # BB#0:
+; SSE2:         pushq %rbp
+; SSE2:         movaps %xmm0, -24(%rsp)
+; SSE2-NEXT:    movzbl -9(%rsp), %eax
+; SSE2-NEXT:    bsrl %eax, %ecx
+; SSE2-NEXT:    movl $15, %eax
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    movzbl -10(%rsp), %ebx
+; SSE2-NEXT:    movzbl -11(%rsp), %edi
+; SSE2-NEXT:    movzbl -12(%rsp), %r9d
+; SSE2-NEXT:    movzbl -13(%rsp), %edx
+; SSE2-NEXT:    movzbl -14(%rsp), %r11d
+; SSE2-NEXT:    movzbl -15(%rsp), %esi
+; SSE2-NEXT:    movzbl -16(%rsp), %r8d
+; SSE2-NEXT:    movzbl -17(%rsp), %ecx
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsrl %edx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movzbl -18(%rsp), %edx
+; SSE2-NEXT:    movzbl -19(%rsp), %ecx
+; SSE2-NEXT:    movzbl -20(%rsp), %r10d
+; SSE2-NEXT:    movzbl -21(%rsp), %ebp
+; SSE2-NEXT:    bsrl %ebp, %ebp
+; SSE2-NEXT:    cmovel %eax, %ebp
+; SSE2-NEXT:    xorl $7, %ebp
+; SSE2-NEXT:    movd %ebp, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    bsrl %edi, %edi
+; SSE2-NEXT:    cmovel %eax, %edi
+; SSE2-NEXT:    xorl $7, %edi
+; SSE2-NEXT:    movd %edi, %xmm1
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsrl %esi, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    movzbl -22(%rsp), %esi
+; SSE2-NEXT:    movzbl -23(%rsp), %ecx
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsrl %ebx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsrl %edx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r11d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsrl %esi, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:    bsrl %r9d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsrl %r10d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r8d, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm4
+; SSE2-NEXT:    movzbl -24(%rsp), %ecx
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    cmovel %eax, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8:
+; SSE3:       # BB#0:
+; SSE3:         pushq %rbp
+; SSE3:         movaps %xmm0, -24(%rsp)
+; SSE3-NEXT:    movzbl -9(%rsp), %eax
+; SSE3-NEXT:    bsrl %eax, %ecx
+; SSE3-NEXT:    movl $15, %eax
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    movzbl -10(%rsp), %ebx
+; SSE3-NEXT:    movzbl -11(%rsp), %edi
+; SSE3-NEXT:    movzbl -12(%rsp), %r9d
+; SSE3-NEXT:    movzbl -13(%rsp), %edx
+; SSE3-NEXT:    movzbl -14(%rsp), %r11d
+; SSE3-NEXT:    movzbl -15(%rsp), %esi
+; SSE3-NEXT:    movzbl -16(%rsp), %r8d
+; SSE3-NEXT:    movzbl -17(%rsp), %ecx
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsrl %edx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    movzbl -18(%rsp), %edx
+; SSE3-NEXT:    movzbl -19(%rsp), %ecx
+; SSE3-NEXT:    movzbl -20(%rsp), %r10d
+; SSE3-NEXT:    movzbl -21(%rsp), %ebp
+; SSE3-NEXT:    bsrl %ebp, %ebp
+; SSE3-NEXT:    cmovel %eax, %ebp
+; SSE3-NEXT:    xorl $7, %ebp
+; SSE3-NEXT:    movd %ebp, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    bsrl %edi, %edi
+; SSE3-NEXT:    cmovel %eax, %edi
+; SSE3-NEXT:    xorl $7, %edi
+; SSE3-NEXT:    movd %edi, %xmm1
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsrl %esi, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    movzbl -22(%rsp), %esi
+; SSE3-NEXT:    movzbl -23(%rsp), %ecx
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsrl %ebx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsrl %edx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r11d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsrl %esi, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE3-NEXT:    bsrl %r9d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsrl %r10d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r8d, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm4
+; SSE3-NEXT:    movzbl -24(%rsp), %ecx
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    cmovel %eax, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    popq %rbp
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8:
+; SSSE3:       # BB#0:
+; SSSE3:         pushq %rbp
+; SSSE3:         movaps %xmm0, -24(%rsp)
+; SSSE3-NEXT:    movzbl -9(%rsp), %eax
+; SSSE3-NEXT:    bsrl %eax, %ecx
+; SSSE3-NEXT:    movl $15, %eax
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    movzbl -10(%rsp), %ebx
+; SSSE3-NEXT:    movzbl -11(%rsp), %edi
+; SSSE3-NEXT:    movzbl -12(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -13(%rsp), %edx
+; SSSE3-NEXT:    movzbl -14(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -15(%rsp), %esi
+; SSSE3-NEXT:    movzbl -16(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -17(%rsp), %ecx
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %edx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movzbl -18(%rsp), %edx
+; SSSE3-NEXT:    movzbl -19(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -20(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -21(%rsp), %ebp
+; SSSE3-NEXT:    bsrl %ebp, %ebp
+; SSSE3-NEXT:    cmovel %eax, %ebp
+; SSSE3-NEXT:    xorl $7, %ebp
+; SSSE3-NEXT:    movd %ebp, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    bsrl %edi, %edi
+; SSSE3-NEXT:    cmovel %eax, %edi
+; SSSE3-NEXT:    xorl $7, %edi
+; SSSE3-NEXT:    movd %edi, %xmm1
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsrl %esi, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    movzbl -22(%rsp), %esi
+; SSSE3-NEXT:    movzbl -23(%rsp), %ecx
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %ebx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsrl %edx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r11d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsrl %esi, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSSE3-NEXT:    bsrl %r9d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsrl %r10d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r8d, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm4
+; SSSE3-NEXT:    movzbl -24(%rsp), %ecx
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    cmovel %eax, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    popq %rbp
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %ecx
+; SSE41-NEXT:    movl $15, %eax
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pextrb $0, %xmm0, %edx
+; SSE41-NEXT:    bsrl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    xorl $7, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrb $1, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $2, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $3, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $4, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $5, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $6, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $7, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $8, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $9, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $10, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $11, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $12, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $13, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $14, %ecx, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    cmovel %eax, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    pinsrb $15, %ecx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %ecx
+; AVX-NEXT:    movl $15, %eax
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX-NEXT:    bsrl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    xorl $7, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrb $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $3, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $4, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $5, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $6, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $9, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $10, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $11, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $12, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    cmovel %eax, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vpinsrb $15, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @testv16i8u(<16 x i8> %in) {
+; SSE2-LABEL: testv16i8u:
+; SSE2:       # BB#0:
+; SSE2:         pushq %rbx
+; SSE2:         movaps %xmm0, -16(%rsp)
+; SSE2-NEXT:    movzbl -1(%rsp), %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -2(%rsp), %edi
+; SSE2-NEXT:    movzbl -3(%rsp), %edx
+; SSE2-NEXT:    movzbl -4(%rsp), %r9d
+; SSE2-NEXT:    movzbl -5(%rsp), %eax
+; SSE2-NEXT:    movzbl -6(%rsp), %r10d
+; SSE2-NEXT:    movzbl -7(%rsp), %ecx
+; SSE2-NEXT:    movzbl -8(%rsp), %r8d
+; SSE2-NEXT:    movzbl -9(%rsp), %esi
+; SSE2-NEXT:    bsrl %esi, %esi
+; SSE2-NEXT:    xorl $7, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -10(%rsp), %eax
+; SSE2-NEXT:    movzbl -11(%rsp), %esi
+; SSE2-NEXT:    movzbl -12(%rsp), %r11d
+; SSE2-NEXT:    movzbl -13(%rsp), %ebx
+; SSE2-NEXT:    bsrl %ebx, %ebx
+; SSE2-NEXT:    xorl $7, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsrl %edx, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    bsrl %esi, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsrl %ecx, %ecx
+; SSE2-NEXT:    xorl $7, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    movzbl -14(%rsp), %ecx
+; SSE2-NEXT:    movzbl -15(%rsp), %edx
+; SSE2-NEXT:    bsrl %edx, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    bsrl %edi, %edx
+; SSE2-NEXT:    xorl $7, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r10d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsrl %ecx, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    bsrl %r9d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsrl %r11d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsrl %r8d, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movzbl -16(%rsp), %eax
+; SSE2-NEXT:    bsrl %eax, %eax
+; SSE2-NEXT:    xorl $7, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8u:
+; SSE3:       # BB#0:
+; SSE3:         pushq %rbx
+; SSE3:         movaps %xmm0, -16(%rsp)
+; SSE3-NEXT:    movzbl -1(%rsp), %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -2(%rsp), %edi
+; SSE3-NEXT:    movzbl -3(%rsp), %edx
+; SSE3-NEXT:    movzbl -4(%rsp), %r9d
+; SSE3-NEXT:    movzbl -5(%rsp), %eax
+; SSE3-NEXT:    movzbl -6(%rsp), %r10d
+; SSE3-NEXT:    movzbl -7(%rsp), %ecx
+; SSE3-NEXT:    movzbl -8(%rsp), %r8d
+; SSE3-NEXT:    movzbl -9(%rsp), %esi
+; SSE3-NEXT:    bsrl %esi, %esi
+; SSE3-NEXT:    xorl $7, %esi
+; SSE3-NEXT:    movd %esi, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -10(%rsp), %eax
+; SSE3-NEXT:    movzbl -11(%rsp), %esi
+; SSE3-NEXT:    movzbl -12(%rsp), %r11d
+; SSE3-NEXT:    movzbl -13(%rsp), %ebx
+; SSE3-NEXT:    bsrl %ebx, %ebx
+; SSE3-NEXT:    xorl $7, %ebx
+; SSE3-NEXT:    movd %ebx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsrl %edx, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    bsrl %esi, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsrl %ecx, %ecx
+; SSE3-NEXT:    xorl $7, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    movzbl -14(%rsp), %ecx
+; SSE3-NEXT:    movzbl -15(%rsp), %edx
+; SSE3-NEXT:    bsrl %edx, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    bsrl %edi, %edx
+; SSE3-NEXT:    xorl $7, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r10d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsrl %ecx, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE3-NEXT:    bsrl %r9d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsrl %r11d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsrl %r8d, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm4
+; SSE3-NEXT:    movzbl -16(%rsp), %eax
+; SSE3-NEXT:    bsrl %eax, %eax
+; SSE3-NEXT:    xorl $7, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8u:
+; SSSE3:       # BB#0:
+; SSSE3:         pushq %rbx
+; SSSE3:         movaps %xmm0, -16(%rsp)
+; SSSE3-NEXT:    movzbl -1(%rsp), %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -2(%rsp), %edi
+; SSSE3-NEXT:    movzbl -3(%rsp), %edx
+; SSSE3-NEXT:    movzbl -4(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -5(%rsp), %eax
+; SSSE3-NEXT:    movzbl -6(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -7(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -8(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -9(%rsp), %esi
+; SSSE3-NEXT:    bsrl %esi, %esi
+; SSSE3-NEXT:    xorl $7, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -10(%rsp), %eax
+; SSSE3-NEXT:    movzbl -11(%rsp), %esi
+; SSSE3-NEXT:    movzbl -12(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -13(%rsp), %ebx
+; SSSE3-NEXT:    bsrl %ebx, %ebx
+; SSSE3-NEXT:    xorl $7, %ebx
+; SSSE3-NEXT:    movd %ebx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsrl %edx, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    bsrl %esi, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %ecx, %ecx
+; SSSE3-NEXT:    xorl $7, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    movzbl -14(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -15(%rsp), %edx
+; SSSE3-NEXT:    bsrl %edx, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    bsrl %edi, %edx
+; SSSE3-NEXT:    xorl $7, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r10d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsrl %ecx, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSSE3-NEXT:    bsrl %r9d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsrl %r11d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsrl %r8d, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm4
+; SSSE3-NEXT:    movzbl -16(%rsp), %eax
+; SSSE3-NEXT:    bsrl %eax, %eax
+; SSSE3-NEXT:    xorl $7, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pextrb $0, %xmm0, %ecx
+; SSE41-NEXT:    bsrl %ecx, %ecx
+; SSE41-NEXT:    xorl $7, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrb $1, %eax, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $4, %eax, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $5, %eax, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $6, %eax, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $7, %eax, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $8, %eax, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $10, %eax, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $11, %eax, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $12, %eax, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $13, %eax, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $14, %eax, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    bsrl %eax, %eax
+; SSE41-NEXT:    xorl $7, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX-NEXT:    bsrl %ecx, %ecx
+; AVX-NEXT:    xorl $7, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $13, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    bsrl %eax, %eax
+; AVX-NEXT:    xorl $7, %eax
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 -1)
+  ret <16 x i8> %out
+}
+
+define <2 x i64> @foldv2i64() {
+; SSE-LABEL: foldv2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $55, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $55, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @foldv2i64u() {
+; SSE-LABEL: foldv2i64u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $55, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $55, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @foldv4i32() {
+; SSE-LABEL: foldv4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [23,0,32,24]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [23,0,32,24]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @foldv4i32u() {
+; SSE-LABEL: foldv4i32u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [23,0,32,24]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [23,0,32,24]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @foldv8i16() {
+; SSE-LABEL: foldv8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @foldv8i16u() {
+; SSE-LABEL: foldv8i16u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @foldv16i8() {
+; SSE-LABEL: foldv16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @foldv16i8u() {
+; SSE-LABEL: foldv16i8u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 -1)
+  ret <16 x i8> %out
+}
+
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-lzcnt-256.ll b/test/CodeGen/X86/vector-lzcnt-256.ll
new file mode 100644
index 0000000000000..48abe1290528d
--- /dev/null
+++ b/test/CodeGen/X86/vector-lzcnt-256.ll
@@ -0,0 +1,1305 @@
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <4 x i64> @testv4i64(<4 x i64> %in) {
+; AVX1-LABEL: testv4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    movl $127, %ecx
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    movl $127, %ecx
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @testv4i64u(<4 x i64> %in) {
+; AVX1-LABEL: testv4i64u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsrq %rax, %rax
+; AVX1-NEXT:    xorq $63, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsrq %rax, %rax
+; AVX2-NEXT:    xorq $63, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @testv8i32(<8 x i32> %in) {
+; AVX1-LABEL: testv8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %ecx
+; AVX1-NEXT:    movl $63, %eax
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $31, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $31, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %ecx
+; AVX2-NEXT:    movl $63, %eax
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $31, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $31, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @testv8i32u(<8 x i32> %in) {
+; AVX1-LABEL: testv8i32u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $31, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $31, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $31, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $31, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @testv16i16(<16 x i16> %in) {
+; AVX1-LABEL: testv16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %cx
+; AVX1-NEXT:    movw $31, %ax
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsrw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    xorl $15, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsrw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    xorl $15, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %cx
+; AVX2-NEXT:    movw $31, %ax
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsrw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    xorl $15, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsrw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    xorl $15, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @testv16i16u(<16 x i16> %in) {
+; AVX1-LABEL: testv16i16u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsrw %cx, %cx
+; AVX1-NEXT:    xorl $15, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX1-NEXT:    bsrw %ax, %ax
+; AVX1-NEXT:    xorl $15, %eax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsrw %cx, %cx
+; AVX2-NEXT:    xorl $15, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX2-NEXT:    bsrw %ax, %ax
+; AVX2-NEXT:    xorl $15, %eax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) {
+; AVX1-LABEL: testv32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %ecx
+; AVX1-NEXT:    movl $15, %eax
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpextrb $0, %xmm1, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $7, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX1-NEXT:    bsrl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    xorl $7, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %ecx
+; AVX2-NEXT:    movl $15, %eax
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpextrb $0, %xmm1, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $7, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpextrb $0, %xmm0, %edx
+; AVX2-NEXT:    bsrl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    xorl $7, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $11, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $12, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vpinsrb $15, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @testv32i8u(<32 x i8> %in) {
+; AVX1-LABEL: testv32i8u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX1-NEXT:    bsrl %ecx, %ecx
+; AVX1-NEXT:    xorl $7, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX1-NEXT:    bsrl %eax, %eax
+; AVX1-NEXT:    xorl $7, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX2-NEXT:    bsrl %ecx, %ecx
+; AVX2-NEXT:    xorl $7, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX2-NEXT:    bsrl %eax, %eax
+; AVX2-NEXT:    xorl $7, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1)
+  ret <32 x i8> %out
+}
+
+define <4 x i64> @foldv4i64() {
+; AVX-LABEL: foldv4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [55,0,64,56]
+; AVX-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @foldv4i64u() {
+; AVX-LABEL: foldv4i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [55,0,64,56]
+; AVX-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @foldv8i32() {
+; AVX-LABEL: foldv8i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]
+; AVX-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @foldv8i32u() {
+; AVX-LABEL: foldv8i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]
+; AVX-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @foldv16i16() {
+; AVX-LABEL: foldv16i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]
+; AVX-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @foldv16i16u() {
+; AVX-LABEL: foldv16i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]
+; AVX-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @foldv32i8() {
+; AVX-LABEL: foldv32i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]
+; AVX-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @foldv32i8u() {
+; AVX-LABEL: foldv32i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]
+; AVX-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 -1)
+  ret <32 x i8> %out
+}
+
+declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-popcnt-128.ll b/test/CodeGen/X86/vector-popcnt-128.ll
new file mode 100644
index 0000000000000..fef445de04ab8
--- /dev/null
+++ b/test/CodeGen/X86/vector-popcnt-128.ll
@@ -0,0 +1,462 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <2 x i64> @testv2i64(<2 x i64> %in) {
+; SSE2-LABEL: testv2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlq $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrlq $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlq $4, %xmm1
+; SSE2-NEXT:    paddq %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlq $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubq %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrlq $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddq %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlq $4, %xmm1
+; SSE3-NEXT:    paddq %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    pxor %xmm0, %xmm0
+; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pshufb %xmm2, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm3
+; SSSE3-NEXT:    paddb %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pshufb %xmm2, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %in)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @testv4i32(<4 x i32> %in) {
+; SSE2-LABEL: testv4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,858993459,858993459]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrld $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $4, %xmm1
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    psadbw %xmm0, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrld $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubd %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [858993459,858993459,858993459,858993459]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrld $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddd %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrld $4, %xmm1
+; SSE3-NEXT:    paddd %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    pxor %xmm0, %xmm0
+; SSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE3-NEXT:    psadbw %xmm0, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    packuswb %xmm2, %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pshufb %xmm3, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm1
+; SSSE3-NEXT:    paddb %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    psadbw %xmm0, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSSE3-NEXT:    packuswb %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    paddb %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE41-NEXT:    psadbw %xmm0, %xmm2
+; SSE41-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE41-NEXT:    psadbw %xmm0, %xmm1
+; SSE41-NEXT:    packuswb %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    vpsadbw %xmm2, %xmm1, %xmm2
+; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %in)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @testv8i16(<8 x i16> %in) {
+; SSE2-LABEL: testv8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubw %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [13107,13107,13107,13107,13107,13107,13107,13107]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddw %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm0
+; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubw %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [13107,13107,13107,13107,13107,13107,13107,13107]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrlw $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddw %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $4, %xmm1
+; SSE3-NEXT:    paddw %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    psllw $8, %xmm0
+; SSE3-NEXT:    paddb %xmm1, %xmm0
+; SSE3-NEXT:    psrlw $8, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pshufb %xmm2, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm3
+; SSSE3-NEXT:    paddb %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    psllw $8, %xmm0
+; SSSE3-NEXT:    paddb %xmm3, %xmm0
+; SSSE3-NEXT:    psrlw $8, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pshufb %xmm2, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    psllw $8, %xmm0
+; SSE41-NEXT:    paddb %xmm3, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsllw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %in)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @testv16i8(<16 x i8> %in) {
+; SSE2-LABEL: testv16i8:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psubb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $1, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    psubb %xmm1, %xmm0
+; SSE3-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; SSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSE3-NEXT:    pand %xmm1, %xmm2
+; SSE3-NEXT:    psrlw $2, %xmm0
+; SSE3-NEXT:    pand %xmm1, %xmm0
+; SSE3-NEXT:    paddb %xmm2, %xmm0
+; SSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSE3-NEXT:    psrlw $4, %xmm1
+; SSE3-NEXT:    paddb %xmm0, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pshufb %xmm3, %xmm4
+; SSSE3-NEXT:    psrlw $4, %xmm0
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pshufb %xmm0, %xmm1
+; SSSE3-NEXT:    paddb %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    paddb %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %in)
+  ret <16 x i8> %out
+}
+
+define <2 x i64> @foldv2i64() {
+; SSE-LABEL: foldv2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,64]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,64]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> <i64 256, i64 -1>)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @foldv4i32() {
+; SSE-LABEL: foldv4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,32,0,8]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,32,0,8]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @foldv8i16() {
+; SSE-LABEL: foldv8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @foldv16i8() {
+; SSE-LABEL: foldv16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>)
+  ret <16 x i8> %out
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
+declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
diff --git a/test/CodeGen/X86/vector-popcnt-256.ll b/test/CodeGen/X86/vector-popcnt-256.ll
new file mode 100644
index 0000000000000..7ce4f712483a2
--- /dev/null
+++ b/test/CodeGen/X86/vector-popcnt-256.ll
@@ -0,0 +1,220 @@
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <4 x i64> @testv4i64(<4 x i64> %in) {
+; AVX1-LABEL: testv4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm5
+; AVX1-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpaddb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpsadbw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpsadbw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %in)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @testv8i32(<8 x i32> %in) {
+; AVX1-LABEL: testv8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX1-NEXT:    vpsadbw %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm5
+; AVX1-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpaddb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX1-NEXT:    vpsadbw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX1-NEXT:    vpsadbw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
+; AVX2-NEXT:    vpsadbw %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX2-NEXT:    vpsadbw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %in)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @testv16i16(<16 x i16> %in) {
+; AVX1-LABEL: testv16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm4
+; AVX1-NEXT:    vpand %xmm1, %xmm4, %xmm4
+; AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsllw $8, %xmm2, %xmm4
+; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpaddb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllw $8, %ymm0, %ymm1
+; AVX2-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %in)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) {
+; AVX1-LABEL: testv32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %in)
+  ret <32 x i8> %out
+}
+
+define <4 x i64> @foldv4i64() {
+; AVX-LABEL: foldv4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,64,0,8]
+; AVX-NEXT:    retq
+  %out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @foldv8i32() {
+; AVX-LABEL: foldv8i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,32,0,8,16,3,2,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @foldv16i16() {
+; AVX-LABEL: foldv16i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [1,16,0,8,0,3,2,3,15,7,1,1,1,1,1,1]
+; AVX-NEXT:    retq
+  %out = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @foldv32i8() {
+; AVX-LABEL: foldv32i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1,1,1,0,0,1,2,3,4,5,6,7,8,2,2,3,7]
+; AVX-NEXT:    retq
+  %out = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>)
+  ret <32 x i8> %out
+}
+
+declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
+declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
+declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)
+declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)
diff --git a/test/CodeGen/X86/vector-shuffle-256-v4.ll b/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 1b42a637907c7..944ec4b8d3ac7 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -843,7 +843,6 @@ define <4 x i64> @insert_mem_and_zero_v4i64(i64* %ptr) {
 define <4 x double> @insert_reg_and_zero_v4f64(double %a) {
 ; ALL-LABEL: insert_reg_and_zero_v4f64:
 ; ALL:       # BB#0:
-; ALL-NEXT:    # kill: XMM0<def> XMM0<kill> YMM0<def>
 ; ALL-NEXT:    vxorpd %ymm1, %ymm1, %ymm1
 ; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3]
 ; ALL-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-shuffle-512-v8.ll b/test/CodeGen/X86/vector-shuffle-512-v8.ll
index 62d4af7809b6b..8dc76231856a2 100644
--- a/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -15,9 +15,8 @@ define <8 x double> @shuffle_v8f64_00000000(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00000010(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00000010:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm1
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0>
   ret <8 x double> %shuffle
@@ -26,9 +25,8 @@ define <8 x double> @shuffle_v8f64_00000010(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00000200(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00000200:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm1
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 2, i32 0, i32 0>
   ret <8 x double> %shuffle
@@ -37,9 +35,8 @@ define <8 x double> @shuffle_v8f64_00000200(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00003000(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00003000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm1
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,0,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0>
   ret <8 x double> %shuffle
@@ -48,11 +45,8 @@ define <8 x double> @shuffle_v8f64_00003000(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00040000(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00040000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vbroadcastsd %xmm1, %ymm1
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; ALL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 4, i32 0, i32 0, i32 0, i32 0>
   ret <8 x double> %shuffle
@@ -61,11 +55,8 @@ define <8 x double> @shuffle_v8f64_00040000(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00500000(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00500000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x double> %shuffle
@@ -74,11 +65,8 @@ define <8 x double> @shuffle_v8f64_00500000(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_06000000(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_06000000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,0]
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 6, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x double> %shuffle
@@ -87,11 +75,11 @@ define <8 x double> @shuffle_v8f64_06000000(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_70000000(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_70000000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[3,0,0,0]
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; ALL-NEXT:    movl $7, %eax
+; ALL-NEXT:    vpinsrq $0, %rax, %xmm1, %xmm2
+; ALL-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x double> %shuffle
@@ -100,10 +88,7 @@ define <8 x double> @shuffle_v8f64_70000000(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_01014545(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_01014545:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermpd $68, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 4, i32 5>
   ret <8 x double> %shuffle
@@ -112,9 +97,8 @@ define <8 x double> @shuffle_v8f64_01014545(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00112233(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00112233:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,0,1,1]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,3,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
   ret <8 x double> %shuffle
@@ -123,9 +107,8 @@ define <8 x double> @shuffle_v8f64_00112233(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00001111(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00001111:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm1
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,1,1,1]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
   ret <8 x double> %shuffle
@@ -134,11 +117,7 @@ define <8 x double> @shuffle_v8f64_00001111(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_81a3c5e7(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_81a3c5e7:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vshufpd $170, %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
   ret <8 x double> %shuffle
@@ -147,10 +126,9 @@ define <8 x double> @shuffle_v8f64_81a3c5e7(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_08080808(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_08080808:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vbroadcastsd %xmm1, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 8, i32 0, i32 8, i32 0, i32 8, i32 0, i32 8>
   ret <8 x double> %shuffle
@@ -159,15 +137,9 @@ define <8 x double> @shuffle_v8f64_08080808(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_08084c4c(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_08084c4c:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vbroadcastsd %xmm3, %ymm3
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3]
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vbroadcastsd %xmm1, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 8, i32 0, i32 8, i32 4, i32 12, i32 4, i32 12>
   ret <8 x double> %shuffle
@@ -176,13 +148,9 @@ define <8 x double> @shuffle_v8f64_08084c4c(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_8823cc67(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_8823cc67:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vbroadcastsd %xmm3, %ymm3
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3]
-; ALL-NEXT:    vbroadcastsd %xmm1, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 8, i32 8, i32 2, i32 3, i32 12, i32 12, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -191,13 +159,9 @@ define <8 x double> @shuffle_v8f64_8823cc67(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_9832dc76(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_9832dc76:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm1[0,1],ymm0[2,3]
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm2 = ymm2[1,0,3,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm1
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3]
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 9, i32 8, i32 3, i32 2, i32 13, i32 12, i32 7, i32 6>
   ret <8 x double> %shuffle
@@ -206,13 +170,9 @@ define <8 x double> @shuffle_v8f64_9832dc76(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_9810dc54(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_9810dc54:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm2
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm2 = ymm2[1,0,3,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm1
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 9, i32 8, i32 1, i32 0, i32 13, i32 12, i32 5, i32 4>
   ret <8 x double> %shuffle
@@ -221,15 +181,9 @@ define <8 x double> @shuffle_v8f64_9810dc54(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_08194c5d(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_08194c5d:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,0,2,1]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
-; ALL-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,1,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   ret <8 x double> %shuffle
@@ -238,15 +192,9 @@ define <8 x double> @shuffle_v8f64_08194c5d(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_2a3b6e7f(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_2a3b6e7f:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
-; ALL-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[2,1,3,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   ret <8 x double> %shuffle
@@ -255,13 +203,9 @@ define <8 x double> @shuffle_v8f64_2a3b6e7f(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_08192a3b(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_08192a3b:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm2 = ymm1[0,2,2,3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[2,1,3,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   ret <8 x double> %shuffle
@@ -270,11 +214,9 @@ define <8 x double> @shuffle_v8f64_08192a3b(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_08991abb(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_08991abb:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm2 = ymm1[0,0,1,1]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm0[0],ymm2[1,2,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,2,3,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 8, i32 9, i32 9, i32 1, i32 10, i32 11, i32 11>
   ret <8 x double> %shuffle
@@ -283,12 +225,9 @@ define <8 x double> @shuffle_v8f64_08991abb(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_091b2d3f(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_091b2d3f:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[2,1,3,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>
   ret <8 x double> %shuffle
@@ -297,11 +236,9 @@ define <8 x double> @shuffle_v8f64_091b2d3f(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_09ab1def(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_09ab1def:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm3 = ymm0[1,0,2,2]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>
   ret <8 x double> %shuffle
@@ -310,10 +247,7 @@ define <8 x double> @shuffle_v8f64_09ab1def(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00014445(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00014445:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,0,0,1]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,0,1]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $64, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 1, i32 4, i32 4, i32 4, i32 5>
   ret <8 x double> %shuffle
@@ -322,10 +256,7 @@ define <8 x double> @shuffle_v8f64_00014445(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00204464(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00204464:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,0,2,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,2,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $32, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 0, i32 4, i32 4, i32 6, i32 4>
   ret <8 x double> %shuffle
@@ -334,10 +265,7 @@ define <8 x double> @shuffle_v8f64_00204464(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_03004744(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_03004744:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,3,0,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $12, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 3, i32 0, i32 0, i32 4, i32 7, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -346,10 +274,7 @@ define <8 x double> @shuffle_v8f64_03004744(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_10005444(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_10005444:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[1,0,0,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 0, i32 0, i32 5, i32 4, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -358,10 +283,7 @@ define <8 x double> @shuffle_v8f64_10005444(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_22006644(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_22006644:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[2,2,0,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $10, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 2, i32 2, i32 0, i32 0, i32 6, i32 6, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -370,10 +292,7 @@ define <8 x double> @shuffle_v8f64_22006644(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_33307774(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_33307774:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[3,3,3,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $63, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 0, i32 7, i32 7, i32 7, i32 4>
   ret <8 x double> %shuffle
@@ -382,10 +301,7 @@ define <8 x double> @shuffle_v8f64_33307774(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_32107654(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_32107654:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,1,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermpd $27, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ret <8 x double> %shuffle
@@ -394,10 +310,7 @@ define <8 x double> @shuffle_v8f64_32107654(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00234467(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00234467:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[0,0,2,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $136, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 3, i32 4, i32 4, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -406,10 +319,7 @@ define <8 x double> @shuffle_v8f64_00234467(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00224466(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00224466:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $0, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   ret <8 x double> %shuffle
@@ -418,10 +328,7 @@ define <8 x double> @shuffle_v8f64_00224466(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_10325476(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_10325476:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,0,3,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $85, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   ret <8 x double> %shuffle
@@ -430,10 +337,7 @@ define <8 x double> @shuffle_v8f64_10325476(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_11335577(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_11335577:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $255, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   ret <8 x double> %shuffle
@@ -442,10 +346,7 @@ define <8 x double> @shuffle_v8f64_11335577(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_10235467(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_10235467:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,0,2,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $153, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -454,10 +355,7 @@ define <8 x double> @shuffle_v8f64_10235467(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_10225466(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_10225466:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,0,2,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $17, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 2, i32 2, i32 5, i32 4, i32 6, i32 6>
   ret <8 x double> %shuffle
@@ -466,10 +364,8 @@ define <8 x double> @shuffle_v8f64_10225466(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00015444(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00015444:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,0,0,1]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 1, i32 5, i32 4, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -478,10 +374,8 @@ define <8 x double> @shuffle_v8f64_00015444(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00204644(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00204644:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,0,2,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 0, i32 4, i32 6, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -490,10 +384,8 @@ define <8 x double> @shuffle_v8f64_00204644(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_03004474(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_03004474:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[0,3,0,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,3,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 3, i32 0, i32 0, i32 4, i32 4, i32 7, i32 4>
   ret <8 x double> %shuffle
@@ -502,10 +394,8 @@ define <8 x double> @shuffle_v8f64_03004474(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_10004444(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_10004444:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[1,0,0,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -514,10 +404,8 @@ define <8 x double> @shuffle_v8f64_10004444(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_22006446(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_22006446:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[2,2,0,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,0,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 2, i32 2, i32 0, i32 0, i32 6, i32 4, i32 4, i32 6>
   ret <8 x double> %shuffle
@@ -526,10 +414,8 @@ define <8 x double> @shuffle_v8f64_22006446(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_33307474(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_33307474:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[3,3,3,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,0,3,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 0, i32 7, i32 4, i32 7, i32 4>
   ret <8 x double> %shuffle
@@ -538,9 +424,8 @@ define <8 x double> @shuffle_v8f64_33307474(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_32104567(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_32104567:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -549,10 +434,8 @@ define <8 x double> @shuffle_v8f64_32104567(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00236744(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00236744:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[0,0,2,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 3, i32 6, i32 7, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -561,10 +444,8 @@ define <8 x double> @shuffle_v8f64_00236744(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00226644(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00226644:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 6, i32 6, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -573,9 +454,7 @@ define <8 x double> @shuffle_v8f64_00226644(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_10324567(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_10324567:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,0,3,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $165, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -584,9 +463,7 @@ define <8 x double> @shuffle_v8f64_10324567(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_11334567(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_11334567:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $175, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -595,9 +472,7 @@ define <8 x double> @shuffle_v8f64_11334567(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_01235467(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_01235467:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[1,0,2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $154, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -606,9 +481,7 @@ define <8 x double> @shuffle_v8f64_01235467(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_01235466(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_01235466:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[1,0,2,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $26, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 6, i32 6>
   ret <8 x double> %shuffle
@@ -617,10 +490,8 @@ define <8 x double> @shuffle_v8f64_01235466(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_002u6u44(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_002u6u44:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,0,0]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 undef, i32 6, i32 undef, i32 4, i32 4>
   ret <8 x double> %shuffle
@@ -629,10 +500,8 @@ define <8 x double> @shuffle_v8f64_002u6u44(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_00uu66uu(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_00uu66uu:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm1
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 6, i32 6, i32 undef, i32 undef>
   ret <8 x double> %shuffle
@@ -641,9 +510,7 @@ define <8 x double> @shuffle_v8f64_00uu66uu(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_103245uu(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_103245uu:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,0,3,2]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $37, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 undef, i32 undef>
   ret <8 x double> %shuffle
@@ -652,9 +519,7 @@ define <8 x double> @shuffle_v8f64_103245uu(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_1133uu67(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_1133uu67:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $143, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 undef, i32 6, i32 7>
   ret <8 x double> %shuffle
@@ -663,9 +528,7 @@ define <8 x double> @shuffle_v8f64_1133uu67(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_0uu354uu(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_0uu354uu:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[1,0,2,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $24, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 5, i32 4, i32 undef, i32 undef>
   ret <8 x double> %shuffle
@@ -674,9 +537,7 @@ define <8 x double> @shuffle_v8f64_0uu354uu(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_uuu3uu66(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_uuu3uu66:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $8, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 6>
   ret <8 x double> %shuffle
@@ -685,16 +546,9 @@ define <8 x double> @shuffle_v8f64_uuu3uu66(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_c348cda0(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_c348cda0:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm2[0,1]
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vbroadcastsd %xmm1, %ymm4
-; ALL-NEXT:    vblendpd {{.*#+}} ymm4 = ymm3[0,1,2],ymm4[3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm2 = ymm4[0],ymm2[1,2],ymm4[3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm3[0,1],ymm1[2],ymm3[3]
-; ALL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 12, i32 3, i32 4, i32 8, i32 12, i32 13, i32 10, i32 0>
   ret <8 x double> %shuffle
@@ -703,17 +557,9 @@ define <8 x double> @shuffle_v8f64_c348cda0(<8 x double> %a, <8 x double> %b) {
 define <8 x double> @shuffle_v8f64_f511235a(<8 x double> %a, <8 x double> %b) {
 ; ALL-LABEL: shuffle_v8f64_f511235a:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vblendpd {{.*#+}} ymm3 = ymm0[0],ymm2[1],ymm0[2,3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[2,3,1,3]
-; ALL-NEXT:    vmovddup {{.*#+}} ymm4 = ymm1[0,0,2,2]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,1]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3]
-; ALL-NEXT:    vextractf64x4 $1, %zmm1, %ymm1
-; ALL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[3,1,2,3]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
-; ALL-NEXT:    vinsertf64x4 $1, %ymm3, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 15, i32 5, i32 1, i32 1, i32 2, i32 3, i32 5, i32 10>
   ret <8 x double> %shuffle
@@ -731,9 +577,8 @@ define <8 x i64> @shuffle_v8i64_00000000(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00000010(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00000010:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm1
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0>
   ret <8 x i64> %shuffle
@@ -742,9 +587,8 @@ define <8 x i64> @shuffle_v8i64_00000010(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00000200(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00000200:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm1
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 2, i32 0, i32 0>
   ret <8 x i64> %shuffle
@@ -753,9 +597,8 @@ define <8 x i64> @shuffle_v8i64_00000200(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00003000(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00003000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm1
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,0,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0>
   ret <8 x i64> %shuffle
@@ -764,11 +607,8 @@ define <8 x i64> @shuffle_v8i64_00003000(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00040000(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00040000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpbroadcastq %xmm1, %ymm1
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm0
-; ALL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 4, i32 0, i32 0, i32 0, i32 0>
   ret <8 x i64> %shuffle
@@ -777,11 +617,8 @@ define <8 x i64> @shuffle_v8i64_00040000(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00500000(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00500000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x i64> %shuffle
@@ -790,11 +627,8 @@ define <8 x i64> @shuffle_v8i64_00500000(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_06000000(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_06000000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,0,0]
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 6, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x i64> %shuffle
@@ -803,11 +637,11 @@ define <8 x i64> @shuffle_v8i64_06000000(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_70000000(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_70000000:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[3,0,0,0]
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; ALL-NEXT:    movl $7, %eax
+; ALL-NEXT:    vpinsrq $0, %rax, %xmm1, %xmm2
+; ALL-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x i64> %shuffle
@@ -816,10 +650,7 @@ define <8 x i64> @shuffle_v8i64_70000000(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_01014545(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_01014545:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm1
-; ALL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermq $68, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 4, i32 5>
   ret <8 x i64> %shuffle
@@ -828,9 +659,8 @@ define <8 x i64> @shuffle_v8i64_01014545(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00112233(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00112233:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,1,1]
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
   ret <8 x i64> %shuffle
@@ -839,9 +669,8 @@ define <8 x i64> @shuffle_v8i64_00112233(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00001111(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00001111:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm1
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i64> %shuffle
@@ -850,11 +679,7 @@ define <8 x i64> @shuffle_v8i64_00001111(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_81a3c5e7(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_81a3c5e7:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vshufpd $170, %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
   ret <8 x i64> %shuffle
@@ -863,10 +688,9 @@ define <8 x i64> @shuffle_v8i64_81a3c5e7(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_08080808(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_08080808:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vpbroadcastq %xmm1, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 8, i32 0, i32 8, i32 0, i32 8, i32 0, i32 8>
   ret <8 x i64> %shuffle
@@ -875,15 +699,9 @@ define <8 x i64> @shuffle_v8i64_08080808(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_08084c4c(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_08084c4c:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vinserti128 $1, %xmm2, %ymm2, %ymm2
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vpbroadcastq %xmm3, %ymm3
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
-; ALL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vpbroadcastq %xmm1, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 8, i32 0, i32 8, i32 4, i32 12, i32 4, i32 12>
   ret <8 x i64> %shuffle
@@ -892,13 +710,9 @@ define <8 x i64> @shuffle_v8i64_08084c4c(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_8823cc67(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_8823cc67:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; ALL-NEXT:    vpbroadcastq %xmm3, %ymm3
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; ALL-NEXT:    vpbroadcastq %xmm1, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 8, i32 8, i32 2, i32 3, i32 12, i32 12, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -907,13 +721,9 @@ define <8 x i64> @shuffle_v8i64_8823cc67(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_9832dc76(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_9832dc76:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; ALL-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; ALL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 9, i32 8, i32 3, i32 2, i32 13, i32 12, i32 7, i32 6>
   ret <8 x i64> %shuffle
@@ -922,13 +732,9 @@ define <8 x i64> @shuffle_v8i64_9832dc76(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_9810dc54(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_9810dc54:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm2
-; ALL-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; ALL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 9, i32 8, i32 1, i32 0, i32 13, i32 12, i32 5, i32 4>
   ret <8 x i64> %shuffle
@@ -937,15 +743,9 @@ define <8 x i64> @shuffle_v8i64_9810dc54(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_08194c5d(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_08194c5d:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; ALL-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,1,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   ret <8 x i64> %shuffle
@@ -954,15 +754,9 @@ define <8 x i64> @shuffle_v8i64_08194c5d(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_2a3b6e7f(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_2a3b6e7f:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; ALL-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,3,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   ret <8 x i64> %shuffle
@@ -971,13 +765,9 @@ define <8 x i64> @shuffle_v8i64_2a3b6e7f(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_08192a3b(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_08192a3b:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[0,2,2,3]
-; ALL-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[2,1,3,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   ret <8 x i64> %shuffle
@@ -986,11 +776,9 @@ define <8 x i64> @shuffle_v8i64_08192a3b(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_08991abb(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_08991abb:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[0,0,1,1]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm0[0,1],ymm2[2,3,4,5,6,7]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,2,3,3]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 8, i32 9, i32 9, i32 1, i32 10, i32 11, i32 11>
   ret <8 x i64> %shuffle
@@ -999,12 +787,9 @@ define <8 x i64> @shuffle_v8i64_08991abb(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_091b2d3f(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_091b2d3f:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[2,1,3,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>
   ret <8 x i64> %shuffle
@@ -1013,11 +798,9 @@ define <8 x i64> @shuffle_v8i64_091b2d3f(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_09ab1def(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_09ab1def:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[2,3,2,3,6,7,6,7]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>
   ret <8 x i64> %shuffle
@@ -1026,10 +809,7 @@ define <8 x i64> @shuffle_v8i64_09ab1def(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00014445(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00014445:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,0,1]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $64, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 1, i32 4, i32 4, i32 4, i32 5>
   ret <8 x i64> %shuffle
@@ -1038,10 +818,7 @@ define <8 x i64> @shuffle_v8i64_00014445(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00204464(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00204464:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,2,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $32, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 0, i32 4, i32 4, i32 6, i32 4>
   ret <8 x i64> %shuffle
@@ -1050,10 +827,7 @@ define <8 x i64> @shuffle_v8i64_00204464(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_03004744(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_03004744:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,3,0,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,3,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $12, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 3, i32 0, i32 0, i32 4, i32 7, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1062,10 +836,7 @@ define <8 x i64> @shuffle_v8i64_03004744(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_10005444(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_10005444:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[1,0,0,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 0, i32 0, i32 5, i32 4, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1074,10 +845,7 @@ define <8 x i64> @shuffle_v8i64_10005444(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_22006644(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_22006644:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,2,0,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $10, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 2, i32 2, i32 0, i32 0, i32 6, i32 6, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1086,10 +854,7 @@ define <8 x i64> @shuffle_v8i64_22006644(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_33307774(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_33307774:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[3,3,3,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,3,3,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $63, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 0, i32 7, i32 7, i32 7, i32 4>
   ret <8 x i64> %shuffle
@@ -1098,10 +863,7 @@ define <8 x i64> @shuffle_v8i64_33307774(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_32107654(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_32107654:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermq $27, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   ret <8 x i64> %shuffle
@@ -1110,10 +872,7 @@ define <8 x i64> @shuffle_v8i64_32107654(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00234467(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00234467:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,2,3]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,3]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $136, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 3, i32 4, i32 4, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1122,10 +881,7 @@ define <8 x i64> @shuffle_v8i64_00234467(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00224466(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00224466:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[0,1,0,1,4,5,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,1,0,1,4,5,4,5]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $0, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   ret <8 x i64> %shuffle
@@ -1134,10 +890,7 @@ define <8 x i64> @shuffle_v8i64_00224466(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_10325476(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_10325476:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $85, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   ret <8 x i64> %shuffle
@@ -1146,10 +899,7 @@ define <8 x i64> @shuffle_v8i64_10325476(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_11335577(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_11335577:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $255, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   ret <8 x i64> %shuffle
@@ -1158,10 +908,7 @@ define <8 x i64> @shuffle_v8i64_11335577(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_10235467(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_10235467:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[1,0,2,3]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,2,3]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $153, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1170,10 +917,7 @@ define <8 x i64> @shuffle_v8i64_10235467(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_10225466(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_10225466:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[1,0,2,2]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,2,2]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $17, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 2, i32 2, i32 5, i32 4, i32 6, i32 6>
   ret <8 x i64> %shuffle
@@ -1182,10 +926,8 @@ define <8 x i64> @shuffle_v8i64_10225466(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00015444(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00015444:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,0,1]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 1, i32 5, i32 4, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1194,10 +936,8 @@ define <8 x i64> @shuffle_v8i64_00015444(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00204644(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00204644:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,2,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 0, i32 4, i32 6, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1206,10 +946,8 @@ define <8 x i64> @shuffle_v8i64_00204644(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_03004474(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_03004474:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,3,0,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,3,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 3, i32 0, i32 0, i32 4, i32 4, i32 7, i32 4>
   ret <8 x i64> %shuffle
@@ -1218,10 +956,8 @@ define <8 x i64> @shuffle_v8i64_03004474(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_10004444(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_10004444:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[1,0,0,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1230,10 +966,8 @@ define <8 x i64> @shuffle_v8i64_10004444(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_22006446(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_22006446:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,2,0,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,0,0,2]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 2, i32 2, i32 0, i32 0, i32 6, i32 4, i32 4, i32 6>
   ret <8 x i64> %shuffle
@@ -1242,10 +976,8 @@ define <8 x i64> @shuffle_v8i64_22006446(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_33307474(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_33307474:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[3,3,3,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,0,3,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 0, i32 7, i32 4, i32 7, i32 4>
   ret <8 x i64> %shuffle
@@ -1254,9 +986,8 @@ define <8 x i64> @shuffle_v8i64_33307474(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_32104567(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_32104567:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1265,10 +996,8 @@ define <8 x i64> @shuffle_v8i64_32104567(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00236744(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00236744:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,0,2,3]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 3, i32 6, i32 7, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1277,10 +1006,8 @@ define <8 x i64> @shuffle_v8i64_00236744(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00226644(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00226644:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[0,1,0,1,4,5,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 6, i32 6, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1289,9 +1016,7 @@ define <8 x i64> @shuffle_v8i64_00226644(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_10324567(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_10324567:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $165, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1300,9 +1025,7 @@ define <8 x i64> @shuffle_v8i64_10324567(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_11334567(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_11334567:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $175, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1311,9 +1034,7 @@ define <8 x i64> @shuffle_v8i64_11334567(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_01235467(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_01235467:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,0,2,3]
-; ALL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $154, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1322,9 +1043,7 @@ define <8 x i64> @shuffle_v8i64_01235467(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_01235466(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_01235466:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,0,2,2]
-; ALL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $26, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 6, i32 6>
   ret <8 x i64> %shuffle
@@ -1333,10 +1052,8 @@ define <8 x i64> @shuffle_v8i64_01235466(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_002u6u44(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_002u6u44:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[0,1,0,1,4,5,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,0,0]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 2, i32 undef, i32 6, i32 undef, i32 4, i32 4>
   ret <8 x i64> %shuffle
@@ -1345,10 +1062,8 @@ define <8 x i64> @shuffle_v8i64_002u6u44(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_00uu66uu(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_00uu66uu:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpbroadcastq %xmm0, %ymm1
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm1
+; ALL-NEXT:    vpermq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 6, i32 6, i32 undef, i32 undef>
   ret <8 x i64> %shuffle
@@ -1357,9 +1072,7 @@ define <8 x i64> @shuffle_v8i64_00uu66uu(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_103245uu(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_103245uu:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $37, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 undef, i32 undef>
   ret <8 x i64> %shuffle
@@ -1368,9 +1081,7 @@ define <8 x i64> @shuffle_v8i64_103245uu(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_1133uu67(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_1133uu67:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7]
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vpermilpd $143, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 undef, i32 6, i32 7>
   ret <8 x i64> %shuffle
@@ -1379,9 +1090,7 @@ define <8 x i64> @shuffle_v8i64_1133uu67(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_0uu354uu(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_0uu354uu:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,3,0,1,6,7,4,5]
-; ALL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $24, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 5, i32 4, i32 undef, i32 undef>
   ret <8 x i64> %shuffle
@@ -1390,9 +1099,7 @@ define <8 x i64> @shuffle_v8i64_0uu354uu(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_uuu3uu66(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_uuu3uu66:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,0,1,4,5,4,5]
-; ALL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; ALL-NEXT:    vpermilpd $8, %zmm0, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 6, i32 6>
   ret <8 x i64> %shuffle
@@ -1401,15 +1108,9 @@ define <8 x i64> @shuffle_v8i64_uuu3uu66(<8 x i64> %a, <8 x i64> %b) {
 define <8 x i64> @shuffle_v8i64_6caa87e5(<8 x i64> %a, <8 x i64> %b) {
 ; ALL-LABEL: shuffle_v8i64_6caa87e5:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; ALL-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; ALL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; ALL-NEXT:    vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3],ymm2[4,5],ymm1[6,7]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; ALL-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,0,1,4,5,4,5]
-; ALL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7]
-; ALL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; ALL-NEXT:    vmovdqa64 {{.*}}(%rip), %zmm2
+; ALL-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; ALL-NEXT:    vmovaps %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 6, i32 12, i32 10, i32 10, i32 8, i32 7, i32 14, i32 5>
   ret <8 x i64> %shuffle
diff --git a/test/CodeGen/X86/vector-tzcnt-128.ll b/test/CodeGen/X86/vector-tzcnt-128.ll
new file mode 100644
index 0000000000000..422fe052d38b9
--- /dev/null
+++ b/test/CodeGen/X86/vector-tzcnt-128.ll
@@ -0,0 +1,1788 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <2 x i64> @testv2i64(<2 x i64> %in) {
+; SSE2-LABEL: testv2i64:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    movl $64, %ecx
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    cmoveq %rcx, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    movl $64, %ecx
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    cmoveq %rcx, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    movl $64, %ecx
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    cmoveq %rcx, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    movl $64, %ecx
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    cmoveq %rcx, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    movl $64, %ecx
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    cmoveq %rcx, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @testv2i64u(<2 x i64> %in) {
+; SSE2-LABEL: testv2i64u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    movd %rax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    bsfq %rax, %rax
+; SSE2-NEXT:    movd %rax, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv2i64u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    movd %rax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %rax
+; SSE3-NEXT:    bsfq %rax, %rax
+; SSE3-NEXT:    movd %rax, %xmm0
+; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv2i64u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    movd %rax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %rax
+; SSSE3-NEXT:    bsfq %rax, %rax
+; SSSE3-NEXT:    movd %rax, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv2i64u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    movd %rax, %xmm1
+; SSE41-NEXT:    movd %xmm0, %rax
+; SSE41-NEXT:    bsfq %rax, %rax
+; SSE41-NEXT:    movd %rax, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    vmovq %rax, %xmm1
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    bsfq %rax, %rax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @testv4i32(<4 x i32> %in) {
+; SSE2-LABEL: testv4i32:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    cmovel %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movl $32, %ecx
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    cmovel %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movl $32, %ecx
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    cmovel %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    movl $32, %ecx
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    cmovel %ecx, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    movl $32, %ecx
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    cmovel %ecx, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @testv4i32u(<4 x i32> %in) {
+; SSE2-LABEL: testv4i32u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv4i32u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE3-NEXT:    movd %xmm1, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm2, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv4i32u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv4i32u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    movd %xmm0, %ecx
+; SSE41-NEXT:    bsfl %ecx, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $1, %eax, %xmm1
+; SSE41-NEXT:    pextrd $2, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrd $2, %eax, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsfl %ecx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @testv8i16(<8 x i16> %in) {
+; SSE2-LABEL: testv8i16:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %cx
+; SSE2-NEXT:    movw $16, %ax
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    bsfw %cx, %cx
+; SSE2-NEXT:    cmovew %ax, %cx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %cx
+; SSE3-NEXT:    movw $16, %ax
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    bsfw %cx, %cx
+; SSE3-NEXT:    cmovew %ax, %cx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %cx
+; SSSE3-NEXT:    movw $16, %ax
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    pextrw $1, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    pextrw $2, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movd %xmm0, %ecx
+; SSSE3-NEXT:    bsfw %cx, %cx
+; SSSE3-NEXT:    cmovew %ax, %cx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $1, %xmm0, %eax
+; SSE41-NEXT:    bsfw %ax, %cx
+; SSE41-NEXT:    movw $16, %ax
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    movd %xmm0, %edx
+; SSE41-NEXT:    bsfw %dx, %dx
+; SSE41-NEXT:    cmovew %ax, %dx
+; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    pinsrw $1, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $2, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $2, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $3, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $3, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $4, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $4, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $5, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $5, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $6, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $6, %ecx, %xmm1
+; SSE41-NEXT:    pextrw $7, %xmm0, %ecx
+; SSE41-NEXT:    bsfw %cx, %cx
+; SSE41-NEXT:    cmovew %ax, %cx
+; SSE41-NEXT:    pinsrw $7, %ecx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %cx
+; AVX-NEXT:    movw $16, %ax
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vmovd %xmm0, %edx
+; AVX-NEXT:    bsfw %dx, %dx
+; AVX-NEXT:    cmovew %ax, %dx
+; AVX-NEXT:    vmovd %edx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $2, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $3, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $5, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    cmovew %ax, %cx
+; AVX-NEXT:    vpinsrw $7, %ecx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @testv8i16u(<8 x i16> %in) {
+; SSE2-LABEL: testv8i16u:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    pextrw $7, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $3, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $5, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $1, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pextrw $6, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    pextrw $4, %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    bsfw %ax, %ax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv8i16u:
+; SSE3:       # BB#0:
+; SSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm1
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    bsfw %ax, %ax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv8i16u:
+; SSSE3:       # BB#0:
+; SSSE3-NEXT:    pextrw $7, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $3, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $5, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSSE3-NEXT:    pextrw $6, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    pextrw $2, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    pextrw $4, %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm1
+; SSSE3-NEXT:    movd %xmm0, %eax
+; SSSE3-NEXT:    bsfw %ax, %ax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv8i16u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:   pextrw $1, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   movd %xmm0, %ecx
+; SSE41-NEXT:   bsfw %cx, %cx
+; SSE41-NEXT:   movd %ecx, %xmm1
+; SSE41-NEXT:   pinsrw $1, %eax, %xmm1
+; SSE41-NEXT:   pextrw $2, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   pinsrw $2, %eax, %xmm1
+; SSE41-NEXT:   pextrw $3, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   pinsrw $3, %eax, %xmm1
+; SSE41-NEXT:   pextrw $4, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   pinsrw $4, %eax, %xmm1
+; SSE41-NEXT:   pextrw $5, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   pinsrw $5, %eax, %xmm1
+; SSE41-NEXT:   pextrw $6, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   pinsrw $6, %eax, %xmm1
+; SSE41-NEXT:   pextrw $7, %xmm0, %eax
+; SSE41-NEXT:   bsfw %ax, %ax
+; SSE41-NEXT:   pinsrw $7, %eax, %xmm1
+; SSE41-NEXT:   movdqa %xmm1, %xmm0
+; SSE41-NEXT:   retq
+;
+; AVX-LABEL: testv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    bsfw %cx, %cx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrw $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX-NEXT:    bsfw %ax, %ax
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @testv16i8(<16 x i8> %in) {
+; SSE2-LABEL: testv16i8:
+; SSE2:       # BB#0:
+; SSE2:         pushq %rbp
+; SSE2:         pushq %r14
+; SSE2:         pushq %rbx
+; SSE2:         movaps %xmm0, -16(%rsp)
+; SSE2-NEXT:    movzbl -1(%rsp), %eax
+; SSE2-NEXT:    bsfl %eax, %edx
+; SSE2-NEXT:    movl $32, %eax
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    movl $8, %ecx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    movzbl -2(%rsp), %r14d
+; SSE2-NEXT:    movzbl -3(%rsp), %ebx
+; SSE2-NEXT:    movzbl -4(%rsp), %r9d
+; SSE2-NEXT:    movzbl -5(%rsp), %edi
+; SSE2-NEXT:    movzbl -6(%rsp), %r11d
+; SSE2-NEXT:    movzbl -7(%rsp), %edx
+; SSE2-NEXT:    movzbl -8(%rsp), %r8d
+; SSE2-NEXT:    movzbl -9(%rsp), %esi
+; SSE2-NEXT:    bsfl %esi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsfl %edi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm2
+; SSE2-NEXT:    movzbl -10(%rsp), %edi
+; SSE2-NEXT:    movzbl -11(%rsp), %esi
+; SSE2-NEXT:    movzbl -12(%rsp), %r10d
+; SSE2-NEXT:    movzbl -13(%rsp), %ebp
+; SSE2-NEXT:    bsfl %ebp, %ebp
+; SSE2-NEXT:    cmovel %eax, %ebp
+; SSE2-NEXT:    cmpl $32, %ebp
+; SSE2-NEXT:    cmovel %ecx, %ebp
+; SSE2-NEXT:    movd %ebp, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    bsfl %ebx, %ebx
+; SSE2-NEXT:    cmovel %eax, %ebx
+; SSE2-NEXT:    cmpl $32, %ebx
+; SSE2-NEXT:    cmovel %ecx, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    bsfl %esi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movzbl -14(%rsp), %edx
+; SSE2-NEXT:    movzbl -15(%rsp), %esi
+; SSE2-NEXT:    bsfl %esi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r14d, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    bsfl %edi, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r11d, %esi
+; SSE2-NEXT:    cmovel %eax, %esi
+; SSE2-NEXT:    cmpl $32, %esi
+; SSE2-NEXT:    cmovel %ecx, %esi
+; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:    bsfl %r9d, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    bsfl %r10d, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r8d, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm4
+; SSE2-NEXT:    movzbl -16(%rsp), %edx
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    cmovel %eax, %edx
+; SSE2-NEXT:    cmpl $32, %edx
+; SSE2-NEXT:    cmovel %ecx, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %r14
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8:
+; SSE3:       # BB#0:
+; SSE3:         pushq %rbp
+; SSE3:         pushq %r14
+; SSE3:         pushq %rbx
+; SSE3:         movaps %xmm0, -16(%rsp)
+; SSE3-NEXT:    movzbl -1(%rsp), %eax
+; SSE3-NEXT:    bsfl %eax, %edx
+; SSE3-NEXT:    movl $32, %eax
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    movl $8, %ecx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    movzbl -2(%rsp), %r14d
+; SSE3-NEXT:    movzbl -3(%rsp), %ebx
+; SSE3-NEXT:    movzbl -4(%rsp), %r9d
+; SSE3-NEXT:    movzbl -5(%rsp), %edi
+; SSE3-NEXT:    movzbl -6(%rsp), %r11d
+; SSE3-NEXT:    movzbl -7(%rsp), %edx
+; SSE3-NEXT:    movzbl -8(%rsp), %r8d
+; SSE3-NEXT:    movzbl -9(%rsp), %esi
+; SSE3-NEXT:    bsfl %esi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsfl %edi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm2
+; SSE3-NEXT:    movzbl -10(%rsp), %edi
+; SSE3-NEXT:    movzbl -11(%rsp), %esi
+; SSE3-NEXT:    movzbl -12(%rsp), %r10d
+; SSE3-NEXT:    movzbl -13(%rsp), %ebp
+; SSE3-NEXT:    bsfl %ebp, %ebp
+; SSE3-NEXT:    cmovel %eax, %ebp
+; SSE3-NEXT:    cmpl $32, %ebp
+; SSE3-NEXT:    cmovel %ecx, %ebp
+; SSE3-NEXT:    movd %ebp, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    bsfl %ebx, %ebx
+; SSE3-NEXT:    cmovel %eax, %ebx
+; SSE3-NEXT:    cmpl $32, %ebx
+; SSE3-NEXT:    cmovel %ecx, %ebx
+; SSE3-NEXT:    movd %ebx, %xmm1
+; SSE3-NEXT:    bsfl %esi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm3
+; SSE3-NEXT:    movzbl -14(%rsp), %edx
+; SSE3-NEXT:    movzbl -15(%rsp), %esi
+; SSE3-NEXT:    bsfl %esi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r14d, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm0
+; SSE3-NEXT:    bsfl %edi, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r11d, %esi
+; SSE3-NEXT:    cmovel %eax, %esi
+; SSE3-NEXT:    cmpl $32, %esi
+; SSE3-NEXT:    cmovel %ecx, %esi
+; SSE3-NEXT:    movd %esi, %xmm0
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE3-NEXT:    bsfl %r9d, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    bsfl %r10d, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r8d, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm4
+; SSE3-NEXT:    movzbl -16(%rsp), %edx
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    cmovel %eax, %edx
+; SSE3-NEXT:    cmpl $32, %edx
+; SSE3-NEXT:    cmovel %ecx, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    popq %r14
+; SSE3-NEXT:    popq %rbp
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8:
+; SSSE3:       # BB#0:
+; SSSE3:         pushq %rbp
+; SSSE3:         pushq %r14
+; SSSE3:         pushq %rbx
+; SSSE3:         movaps %xmm0, -16(%rsp)
+; SSSE3-NEXT:    movzbl -1(%rsp), %eax
+; SSSE3-NEXT:    bsfl %eax, %edx
+; SSSE3-NEXT:    movl $32, %eax
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    movl $8, %ecx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    movzbl -2(%rsp), %r14d
+; SSSE3-NEXT:    movzbl -3(%rsp), %ebx
+; SSSE3-NEXT:    movzbl -4(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -5(%rsp), %edi
+; SSSE3-NEXT:    movzbl -6(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -7(%rsp), %edx
+; SSSE3-NEXT:    movzbl -8(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -9(%rsp), %esi
+; SSSE3-NEXT:    bsfl %esi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %edi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm2
+; SSSE3-NEXT:    movzbl -10(%rsp), %edi
+; SSSE3-NEXT:    movzbl -11(%rsp), %esi
+; SSSE3-NEXT:    movzbl -12(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -13(%rsp), %ebp
+; SSSE3-NEXT:    bsfl %ebp, %ebp
+; SSSE3-NEXT:    cmovel %eax, %ebp
+; SSSE3-NEXT:    cmpl $32, %ebp
+; SSSE3-NEXT:    cmovel %ecx, %ebp
+; SSSE3-NEXT:    movd %ebp, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    bsfl %ebx, %ebx
+; SSSE3-NEXT:    cmovel %eax, %ebx
+; SSSE3-NEXT:    cmpl $32, %ebx
+; SSSE3-NEXT:    cmovel %ecx, %ebx
+; SSSE3-NEXT:    movd %ebx, %xmm1
+; SSSE3-NEXT:    bsfl %esi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    movzbl -14(%rsp), %edx
+; SSSE3-NEXT:    movzbl -15(%rsp), %esi
+; SSSE3-NEXT:    bsfl %esi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r14d, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    bsfl %edi, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r11d, %esi
+; SSSE3-NEXT:    cmovel %eax, %esi
+; SSSE3-NEXT:    cmpl $32, %esi
+; SSSE3-NEXT:    cmovel %ecx, %esi
+; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSSE3-NEXT:    bsfl %r9d, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    bsfl %r10d, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r8d, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm4
+; SSSE3-NEXT:    movzbl -16(%rsp), %edx
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    cmovel %eax, %edx
+; SSSE3-NEXT:    cmpl $32, %edx
+; SSSE3-NEXT:    cmovel %ecx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    popq %r14
+; SSSE3-NEXT:    popq %rbp
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %edx
+; SSE41-NEXT:    movl $32, %eax
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    movl $8, %ecx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pextrb $0, %xmm0, %esi
+; SSE41-NEXT:    bsfl %esi, %esi
+; SSE41-NEXT:    cmovel %eax, %esi
+; SSE41-NEXT:    cmpl $32, %esi
+; SSE41-NEXT:    cmovel %ecx, %esi
+; SSE41-NEXT:    movd %esi, %xmm1
+; SSE41-NEXT:    pinsrb $1, %edx, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $2, %edx, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $3, %edx, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $4, %edx, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $5, %edx, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $6, %edx, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $7, %edx, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $8, %edx, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $9, %edx, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $10, %edx, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $11, %edx, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $12, %edx, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $13, %edx, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $14, %edx, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %edx
+; SSE41-NEXT:    bsfl %edx, %edx
+; SSE41-NEXT:    cmovel %eax, %edx
+; SSE41-NEXT:    cmpl $32, %edx
+; SSE41-NEXT:    cmovel %ecx, %edx
+; SSE41-NEXT:    pinsrb $15, %edx, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %edx
+; AVX-NEXT:    movl $32, %eax
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    movl $8, %ecx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpextrb $0, %xmm0, %esi
+; AVX-NEXT:    bsfl %esi, %esi
+; AVX-NEXT:    cmovel %eax, %esi
+; AVX-NEXT:    cmpl $32, %esi
+; AVX-NEXT:    cmovel %ecx, %esi
+; AVX-NEXT:    vmovd %esi, %xmm1
+; AVX-NEXT:    vpinsrb $1, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $3, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $4, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $5, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $7, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $8, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $9, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $10, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $11, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $12, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $13, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $14, %edx, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %edx
+; AVX-NEXT:    bsfl %edx, %edx
+; AVX-NEXT:    cmovel %eax, %edx
+; AVX-NEXT:    cmpl $32, %edx
+; AVX-NEXT:    cmovel %ecx, %edx
+; AVX-NEXT:    vpinsrb $15, %edx, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @testv16i8u(<16 x i8> %in) {
+; SSE2-LABEL: testv16i8u:
+; SSE2:       # BB#0:
+; SSE2:         pushq %rbx
+; SSE2:         movaps %xmm0, -16(%rsp)
+; SSE2-NEXT:    movzbl -1(%rsp), %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -2(%rsp), %r11d
+; SSE2-NEXT:    movzbl -3(%rsp), %eax
+; SSE2-NEXT:    movzbl -4(%rsp), %r9d
+; SSE2-NEXT:    movzbl -5(%rsp), %edi
+; SSE2-NEXT:    movzbl -6(%rsp), %r10d
+; SSE2-NEXT:    movzbl -7(%rsp), %ecx
+; SSE2-NEXT:    movzbl -8(%rsp), %r8d
+; SSE2-NEXT:    movzbl -9(%rsp), %edx
+; SSE2-NEXT:    bsfl %edx, %edx
+; SSE2-NEXT:    movd %edx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    bsfl %edi, %edx
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    movzbl -10(%rsp), %edx
+; SSE2-NEXT:    movzbl -11(%rsp), %esi
+; SSE2-NEXT:    movzbl -12(%rsp), %edi
+; SSE2-NEXT:    movzbl -13(%rsp), %ebx
+; SSE2-NEXT:    bsfl %ebx, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsfl %esi, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw %xmm0, %xmm3    # xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    bsfl %ecx, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl -14(%rsp), %eax
+; SSE2-NEXT:    movzbl -15(%rsp), %ecx
+; SSE2-NEXT:    bsfl %ecx, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    bsfl %r11d, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsfl %edx, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r10d, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    bsfl %r9d, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    bsfl %edi, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    bsfl %r8d, %eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movzbl -16(%rsp), %eax
+; SSE2-NEXT:    bsfl %eax, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    retq
+;
+; SSE3-LABEL: testv16i8u:
+; SSE3:       # BB#0:
+; SSE3:         pushq %rbx
+; SSE3:         movaps %xmm0, -16(%rsp)
+; SSE3-NEXT:    movzbl -1(%rsp), %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -2(%rsp), %r11d
+; SSE3-NEXT:    movzbl -3(%rsp), %eax
+; SSE3-NEXT:    movzbl -4(%rsp), %r9d
+; SSE3-NEXT:    movzbl -5(%rsp), %edi
+; SSE3-NEXT:    movzbl -6(%rsp), %r10d
+; SSE3-NEXT:    movzbl -7(%rsp), %ecx
+; SSE3-NEXT:    movzbl -8(%rsp), %r8d
+; SSE3-NEXT:    movzbl -9(%rsp), %edx
+; SSE3-NEXT:    bsfl %edx, %edx
+; SSE3-NEXT:    movd %edx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    bsfl %edi, %edx
+; SSE3-NEXT:    movd %edx, %xmm0
+; SSE3-NEXT:    movzbl -10(%rsp), %edx
+; SSE3-NEXT:    movzbl -11(%rsp), %esi
+; SSE3-NEXT:    movzbl -12(%rsp), %edi
+; SSE3-NEXT:    movzbl -13(%rsp), %ebx
+; SSE3-NEXT:    bsfl %ebx, %ebx
+; SSE3-NEXT:    movd %ebx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsfl %esi, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklbw %xmm0, %xmm3    # xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    bsfl %ecx, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    movzbl -14(%rsp), %eax
+; SSE3-NEXT:    movzbl -15(%rsp), %ecx
+; SSE3-NEXT:    bsfl %ecx, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm1
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE3-NEXT:    bsfl %r11d, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsfl %edx, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r10d, %ecx
+; SSE3-NEXT:    movd %ecx, %xmm0
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm3
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE3-NEXT:    bsfl %r9d, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    bsfl %edi, %eax
+; SSE3-NEXT:    movd %eax, %xmm2
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE3-NEXT:    bsfl %r8d, %eax
+; SSE3-NEXT:    movd %eax, %xmm4
+; SSE3-NEXT:    movzbl -16(%rsp), %eax
+; SSE3-NEXT:    bsfl %eax, %eax
+; SSE3-NEXT:    movd %eax, %xmm0
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE3-NEXT:    popq %rbx
+; SSE3-NEXT:    retq
+;
+; SSSE3-LABEL: testv16i8u:
+; SSSE3:       # BB#0:
+; SSSE3:         pushq %rbx
+; SSSE3:         movaps %xmm0, -16(%rsp)
+; SSSE3-NEXT:    movzbl -1(%rsp), %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -2(%rsp), %r11d
+; SSSE3-NEXT:    movzbl -3(%rsp), %eax
+; SSSE3-NEXT:    movzbl -4(%rsp), %r9d
+; SSSE3-NEXT:    movzbl -5(%rsp), %edi
+; SSSE3-NEXT:    movzbl -6(%rsp), %r10d
+; SSSE3-NEXT:    movzbl -7(%rsp), %ecx
+; SSSE3-NEXT:    movzbl -8(%rsp), %r8d
+; SSSE3-NEXT:    movzbl -9(%rsp), %edx
+; SSSE3-NEXT:    bsfl %edx, %edx
+; SSSE3-NEXT:    movd %edx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %edi, %edx
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    movzbl -10(%rsp), %edx
+; SSSE3-NEXT:    movzbl -11(%rsp), %esi
+; SSSE3-NEXT:    movzbl -12(%rsp), %edi
+; SSSE3-NEXT:    movzbl -13(%rsp), %ebx
+; SSSE3-NEXT:    bsfl %ebx, %ebx
+; SSSE3-NEXT:    movd %ebx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsfl %esi, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklbw %xmm0, %xmm3    # xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %ecx, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl -14(%rsp), %eax
+; SSSE3-NEXT:    movzbl -15(%rsp), %ecx
+; SSSE3-NEXT:    bsfl %ecx, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSSE3-NEXT:    bsfl %r11d, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsfl %edx, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r10d, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSSE3-NEXT:    bsfl %r9d, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    bsfl %edi, %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    bsfl %r8d, %eax
+; SSSE3-NEXT:    movd %eax, %xmm4
+; SSSE3-NEXT:    movzbl -16(%rsp), %eax
+; SSSE3-NEXT:    bsfl %eax, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: testv16i8u:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrb $1, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pextrb $0, %xmm0, %ecx
+; SSE41-NEXT:    bsfl %ecx, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm1
+; SSE41-NEXT:    pinsrb $1, %eax, %xmm1
+; SSE41-NEXT:    pextrb $2, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $2, %eax, %xmm1
+; SSE41-NEXT:    pextrb $3, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $3, %eax, %xmm1
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $4, %eax, %xmm1
+; SSE41-NEXT:    pextrb $5, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $5, %eax, %xmm1
+; SSE41-NEXT:    pextrb $6, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $6, %eax, %xmm1
+; SSE41-NEXT:    pextrb $7, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $7, %eax, %xmm1
+; SSE41-NEXT:    pextrb $8, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $8, %eax, %xmm1
+; SSE41-NEXT:    pextrb $9, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm1
+; SSE41-NEXT:    pextrb $10, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $10, %eax, %xmm1
+; SSE41-NEXT:    pextrb $11, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $11, %eax, %xmm1
+; SSE41-NEXT:    pextrb $12, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $12, %eax, %xmm1
+; SSE41-NEXT:    pextrb $13, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $13, %eax, %xmm1
+; SSE41-NEXT:    pextrb $14, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $14, %eax, %xmm1
+; SSE41-NEXT:    pextrb $15, %xmm0, %eax
+; SSE41-NEXT:    bsfl %eax, %eax
+; SSE41-NEXT:    pinsrb $15, %eax, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: testv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX-NEXT:    bsfl %ecx, %ecx
+; AVX-NEXT:    vmovd %ecx, %xmm1
+; AVX-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $2, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $3, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $4, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $5, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $6, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $13, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
+; AVX-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX-NEXT:    bsfl %eax, %eax
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 -1)
+  ret <16 x i8> %out
+}
+
+define <2 x i64> @foldv2i64() {
+; SSE-LABEL: foldv2i64:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $8, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $8, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 0)
+  ret <2 x i64> %out
+}
+
+define <2 x i64> @foldv2i64u() {
+; SSE-LABEL: foldv2i64u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movl $8, %eax
+; SSE-NEXT:    movd %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv2i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    movl $8, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> <i64 256, i64 -1>, i1 -1)
+  ret <2 x i64> %out
+}
+
+define <4 x i32> @foldv4i32() {
+; SSE-LABEL: foldv4i32:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,32,0]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 0)
+  ret <4 x i32> %out
+}
+
+define <4 x i32> @foldv4i32u() {
+; SSE-LABEL: foldv4i32u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,32,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv4i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,32,0]
+; AVX-NEXT:    retq
+  %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> <i32 256, i32 -1, i32 0, i32 255>, i1 -1)
+  ret <4 x i32> %out
+}
+
+define <8 x i16> @foldv8i16() {
+; SSE-LABEL: foldv8i16:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 0)
+  ret <8 x i16> %out
+}
+
+define <8 x i16> @foldv8i16u() {
+; SSE-LABEL: foldv8i16u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv8i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88>, i1 -1)
+  ret <8 x i16> %out
+}
+
+define <16 x i8> @foldv16i8() {
+; SSE-LABEL: foldv16i8:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 0)
+  ret <16 x i8> %out
+}
+
+define <16 x i8> @foldv16i8u() {
+; SSE-LABEL: foldv16i8u:
+; SSE:       # BB#0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: foldv16i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5]
+; AVX-NEXT:    retq
+  %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32>, i1 -1)
+  ret <16 x i8> %out
+}
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-tzcnt-256.ll b/test/CodeGen/X86/vector-tzcnt-256.ll
new file mode 100644
index 0000000000000..8f744f79f85fa
--- /dev/null
+++ b/test/CodeGen/X86/vector-tzcnt-256.ll
@@ -0,0 +1,1195 @@
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+
+target triple = "x86_64-unknown-unknown"
+
+define <4 x i64> @testv4i64(<4 x i64> %in) {
+; AVX1-LABEL: testv4i64:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    movl $64, %ecx
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    cmoveq %rcx, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    movl $64, %ecx
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    cmoveq %rcx, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @testv4i64u(<4 x i64> %in) {
+; AVX1-LABEL: testv4i64u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm1
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    bsfq %rax, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv4i64u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm1
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %xmm0, %rax
+; AVX2-NEXT:    bsfq %rax, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @testv8i32(<8 x i32> %in) {
+; AVX1-LABEL: testv8i32:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %ecx
+; AVX1-NEXT:    movl $32, %eax
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    cmovel %eax, %ecx
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %ecx
+; AVX2-NEXT:    movl $32, %eax
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    cmovel %eax, %ecx
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @testv8i32u(<8 x i32> %in) {
+; AVX1-LABEL: testv8i32u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv8i32u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @testv16i16(<16 x i16> %in) {
+; AVX1-LABEL: testv16i16:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %cx
+; AVX1-NEXT:    movw $16, %ax
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vmovd %xmm1, %edx
+; AVX1-NEXT:    bsfw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vmovd %xmm0, %edx
+; AVX1-NEXT:    bsfw %dx, %dx
+; AVX1-NEXT:    cmovew %ax, %dx
+; AVX1-NEXT:    vmovd %edx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    cmovew %ax, %cx
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %cx
+; AVX2-NEXT:    movw $16, %ax
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    bsfw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vmovd %xmm0, %edx
+; AVX2-NEXT:    bsfw %dx, %dx
+; AVX2-NEXT:    cmovew %ax, %dx
+; AVX2-NEXT:    vmovd %edx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    cmovew %ax, %cx
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @testv16i16u(<16 x i16> %in) {
+; AVX1-LABEL: testv16i16u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vmovd %xmm1, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vmovd %xmm0, %ecx
+; AVX1-NEXT:    bsfw %cx, %cx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX1-NEXT:    bsfw %ax, %ax
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv16i16u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vmovd %xmm1, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm1, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    bsfw %cx, %cx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
+; AVX2-NEXT:    bsfw %ax, %ax
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @testv32i8(<32 x i8> %in) {
+; AVX1-LABEL: testv32i8:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %edx
+; AVX1-NEXT:    movl $32, %eax
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    movl $8, %ecx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpextrb $0, %xmm1, %esi
+; AVX1-NEXT:    bsfl %esi, %esi
+; AVX1-NEXT:    cmovel %eax, %esi
+; AVX1-NEXT:    cmpl $32, %esi
+; AVX1-NEXT:    cmovel %ecx, %esi
+; AVX1-NEXT:    vmovd %esi, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpextrb $0, %xmm0, %esi
+; AVX1-NEXT:    bsfl %esi, %esi
+; AVX1-NEXT:    cmovel %eax, %esi
+; AVX1-NEXT:    cmpl $32, %esi
+; AVX1-NEXT:    cmovel %ecx, %esi
+; AVX1-NEXT:    vmovd %esi, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %edx
+; AVX1-NEXT:    bsfl %edx, %edx
+; AVX1-NEXT:    cmovel %eax, %edx
+; AVX1-NEXT:    cmpl $32, %edx
+; AVX1-NEXT:    cmovel %ecx, %edx
+; AVX1-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %edx
+; AVX2-NEXT:    movl $32, %eax
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    movl $8, %ecx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpextrb $0, %xmm1, %esi
+; AVX2-NEXT:    bsfl %esi, %esi
+; AVX2-NEXT:    cmovel %eax, %esi
+; AVX2-NEXT:    cmpl $32, %esi
+; AVX2-NEXT:    cmovel %ecx, %esi
+; AVX2-NEXT:    vmovd %esi, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpextrb $0, %xmm0, %esi
+; AVX2-NEXT:    bsfl %esi, %esi
+; AVX2-NEXT:    cmovel %eax, %esi
+; AVX2-NEXT:    cmpl $32, %esi
+; AVX2-NEXT:    cmovel %ecx, %esi
+; AVX2-NEXT:    vmovd %esi, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $5, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $7, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $8, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $9, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $10, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $11, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $12, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $13, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $14, %edx, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %edx
+; AVX2-NEXT:    bsfl %edx, %edx
+; AVX2-NEXT:    cmovel %eax, %edx
+; AVX2-NEXT:    cmpl $32, %edx
+; AVX2-NEXT:    cmovel %ecx, %edx
+; AVX2-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @testv32i8u(<32 x i8> %in) {
+; AVX1-LABEL: testv32i8u:
+; AVX1:       # BB#0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX1-NEXT:    bsfl %ecx, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm2
+; AVX1-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX1-NEXT:    bsfl %eax, %eax
+; AVX1-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: testv32i8u:
+; AVX2:       # BB#0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm1, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm1, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpextrb $0, %xmm0, %ecx
+; AVX2-NEXT:    bsfl %ecx, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm2
+; AVX2-NEXT:    vpinsrb $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $2, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $3, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $4, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $5, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $6, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $7, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $8, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $10, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $11, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $12, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $13, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $13, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $14, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpextrb $15, %xmm0, %eax
+; AVX2-NEXT:    bsfl %eax, %eax
+; AVX2-NEXT:    vpinsrb $15, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 -1)
+  ret <32 x i8> %out
+}
+
+define <4 x i64> @foldv4i64() {
+; AVX-LABEL: foldv4i64:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,64,0]
+; AVX-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 0)
+  ret <4 x i64> %out
+}
+
+define <4 x i64> @foldv4i64u() {
+; AVX-LABEL: foldv4i64u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,64,0]
+; AVX-NEXT:    retq
+  %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 -1)
+  ret <4 x i64> %out
+}
+
+define <8 x i32> @foldv8i32() {
+; AVX-LABEL: foldv8i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 0)
+  ret <8 x i32> %out
+}
+
+define <8 x i32> @foldv8i32u() {
+; AVX-LABEL: foldv8i32u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3]
+; AVX-NEXT:    retq
+  %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 -1)
+  ret <8 x i32> %out
+}
+
+define <16 x i16> @foldv16i16() {
+; AVX-LABEL: foldv16i16:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
+; AVX-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 0)
+  ret <16 x i16> %out
+}
+
+define <16 x i16> @foldv16i16u() {
+; AVX-LABEL: foldv16i16u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5]
+; AVX-NEXT:    retq
+  %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 -1)
+  ret <16 x i16> %out
+}
+
+define <32 x i8> @foldv32i8() {
+; AVX-LABEL: foldv32i8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
+; AVX-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 0)
+  ret <32 x i8> %out
+}
+
+define <32 x i8> @foldv32i8u() {
+; AVX-LABEL: foldv32i8u:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0]
+; AVX-NEXT:    retq
+  %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 -1)
+  ret <32 x i8> %out
+}
+
+declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
diff --git a/test/CodeGen/X86/vector-zext.ll b/test/CodeGen/X86/vector-zext.ll
index 42781830ff2fb..c64e17442675e 100644
--- a/test/CodeGen/X86/vector-zext.ll
+++ b/test/CodeGen/X86/vector-zext.ll
@@ -9,7 +9,6 @@ define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pand .LCPI0_0(%rip), %xmm1
@@ -19,7 +18,6 @@ define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pand .LCPI0_0(%rip), %xmm1
@@ -156,7 +154,6 @@ define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %z) {
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    pand .LCPI3_0(%rip), %xmm1
@@ -166,7 +163,6 @@ define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %z) {
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSSE3-NEXT:    pand .LCPI3_0(%rip), %xmm1
@@ -334,7 +330,6 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE2-NEXT:    retq
@@ -343,7 +338,6 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSSE3-NEXT:    retq
@@ -366,7 +360,6 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone
 ;
 ; AVX2-LABEL: shuf_zext_8i16_to_8i32:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    # kill
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    retq
 entry:
@@ -380,7 +373,6 @@ define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    # kill
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
@@ -389,7 +381,6 @@ define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    # kill
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
@@ -413,7 +404,6 @@ define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone
 ;
 ; AVX2-LABEL: shuf_zext_4i32_to_4i64:
 ; AVX2:       # BB#0: # %entry
-; AVX2-NEXT:    # kill
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    retq
 entry:
diff --git a/test/CodeGen/X86/win32-eh-states.ll b/test/CodeGen/X86/win32-eh-states.ll
new file mode 100644
index 0000000000000..8db127df6da73
--- /dev/null
+++ b/test/CodeGen/X86/win32-eh-states.ll
@@ -0,0 +1,112 @@
+; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s
+
+; Based on this source:
+; extern "C" void may_throw(int);
+; void f() {
+;   try {
+;     may_throw(1);
+;     try {
+;       may_throw(2);
+;     } catch (int) {
+;       may_throw(3);
+;     }
+;   } catch (int) {
+;     may_throw(4);
+;   }
+; }
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchHandlerType = type { i32, i8* }
+
+declare void @may_throw(i32)
+declare i32 @__CxxFrameHandler3(...)
+declare void @llvm.eh.begincatch(i8*, i8*)
+declare void @llvm.eh.endcatch()
+declare i32 @llvm.eh.typeid.for(i8*)
+
+$"\01??_R0H@8" = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@llvm.eh.handlertype.H.0 = private unnamed_addr constant %eh.CatchHandlerType { i32 0, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*) }, section "llvm.metadata"
+
+define void @f() #0 {
+entry:
+  invoke void @may_throw(i32 1)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @may_throw(i32 2)
+          to label %try.cont.9 unwind label %lpad.1
+
+try.cont.9:                                       ; preds = %invoke.cont.3, %invoke.cont, %catch.7
+  ; FIXME: Something about our CFG breaks TailDuplication. This empy asm blocks
+  ; it so we can focus on testing the state numbering.
+  call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"()
+  ret void
+
+lpad:                                             ; preds = %catch, %entry
+  %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+          catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0
+  %1 = extractvalue { i8*, i32 } %0, 0
+  %2 = extractvalue { i8*, i32 } %0, 1
+  br label %catch.dispatch.4
+
+lpad.1:                                           ; preds = %invoke.cont
+  %3 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+          catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)
+  %4 = extractvalue { i8*, i32 } %3, 0
+  %5 = extractvalue { i8*, i32 } %3, 1
+  %6 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3
+  %matches = icmp eq i32 %5, %6
+  br i1 %matches, label %catch, label %catch.dispatch.4
+
+catch.dispatch.4:                                 ; preds = %lpad.1, %lpad
+  %exn.slot.0 = phi i8* [ %4, %lpad.1 ], [ %1, %lpad ]
+  %ehselector.slot.0 = phi i32 [ %5, %lpad.1 ], [ %2, %lpad ]
+  %.pre = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3
+  %matches6 = icmp eq i32 %ehselector.slot.0, %.pre
+  br i1 %matches6, label %catch.7, label %eh.resume
+
+catch.7:                                          ; preds = %catch.dispatch.4
+  tail call void @llvm.eh.begincatch(i8* %exn.slot.0, i8* null) #3
+  tail call void @may_throw(i32 4)
+  tail call void @llvm.eh.endcatch() #3
+  br label %try.cont.9
+
+catch:                                            ; preds = %lpad.1
+  tail call void @llvm.eh.begincatch(i8* %4, i8* null) #3
+  invoke void @may_throw(i32 3)
+          to label %invoke.cont.3 unwind label %lpad
+
+invoke.cont.3:                                    ; preds = %catch
+  tail call void @llvm.eh.endcatch() #3
+  br label %try.cont.9
+
+eh.resume:                                        ; preds = %catch.dispatch.4
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn.slot.0, 0
+  %lpad.val.12 = insertvalue { i8*, i32 } %lpad.val, i32 %ehselector.slot.0, 1
+  resume { i8*, i32 } %lpad.val.12
+}
+
+; CHECK-LABEL: _f:
+; CHECK: movl $-1, [[state:[-0-9]+]](%ebp)
+; CHECK: movl $___ehhandler$f, {{.*}}
+;
+; CHECK: movl $0, [[state]](%ebp)
+; CHECK: movl $1, (%esp)
+; CHECK: calll _may_throw
+;
+; CHECK: movl $1, [[state]](%ebp)
+; CHECK: movl $2, (%esp)
+; CHECK: calll _may_throw
+
+; CHECK-LABEL: _f.catch:
+; CHECK: movl $4, Lf$frame_escape_{{[0-9]+.*}}
+; CHECK: movl $4, (%esp)
+; CHECK: calll _may_throw
+
+; CHECK-LABEL: _f.catch.1:
+; CHECK: movl $3, Lf$frame_escape_{{[0-9]+.*}}
+; CHECK: movl $3, (%esp)
+; CHECK: calll _may_throw
diff --git a/test/CodeGen/X86/win32-eh.ll b/test/CodeGen/X86/win32-eh.ll
index 4d3c34ed6c175..42c9d9e2240de 100644
--- a/test/CodeGen/X86/win32-eh.ll
+++ b/test/CodeGen/X86/win32-eh.ll
@@ -19,16 +19,18 @@ catchall:
 }
 
 ; CHECK-LABEL: _use_except_handler3:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
 ; CHECK: subl ${{[0-9]+}}, %esp
-; CHECK: movl $-1, 12(%esp)
-; CHECK: movl $L__ehtable$use_except_handler3, 8(%esp)
-; CHECK: movl $__except_handler3, 4(%esp)
+; CHECK: movl $-1, -4(%ebp)
+; CHECK: movl $L__ehtable$use_except_handler3, -8(%ebp)
+; CHECK: leal -16(%ebp), %[[node:[^ ,]*]]
+; CHECK: movl $__except_handler3, -12(%ebp)
 ; CHECK: movl %fs:0, %[[next:[^ ,]*]]
-; CHECK: movl %[[next]], (%esp)
-; CHECK: leal (%esp), %[[node:[^ ,]*]]
+; CHECK: movl %[[next]], -16(%ebp)
 ; CHECK: movl %[[node]], %fs:0
 ; CHECK: calll _may_throw_or_crash
-; CHECK: movl (%esp), %[[next:[^ ,]*]]
+; CHECK: movl -16(%ebp), %[[next:[^ ,]*]]
 ; CHECK: movl %[[next]], %fs:0
 ; CHECK: retl
 
@@ -44,17 +46,21 @@ catchall:
 }
 
 ; CHECK-LABEL: _use_except_handler4:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
 ; CHECK: subl ${{[0-9]+}}, %esp
-; CHECK: movl %esp, (%esp)
-; CHECK: movl $-2, 20(%esp)
-; CHECK: movl $L__ehtable$use_except_handler4, 4(%esp)
-; CHECK: leal 8(%esp), %[[node:[^ ,]*]]
-; CHECK: movl $__except_handler4, 12(%esp)
+; CHECK: movl %esp, -24(%ebp)
+; CHECK: movl $-2, -4(%ebp)
+; CHECK: movl $L__ehtable$use_except_handler4, %[[lsda:[^ ,]*]]
+; CHECK: xorl ___security_cookie, %[[lsda]]
+; CHECK: movl %[[lsda]], -8(%ebp)
+; CHECK: leal -16(%ebp), %[[node:[^ ,]*]]
+; CHECK: movl $__except_handler4, -12(%ebp)
 ; CHECK: movl %fs:0, %[[next:[^ ,]*]]
-; CHECK: movl %[[next]], 8(%esp)
+; CHECK: movl %[[next]], -16(%ebp)
 ; CHECK: movl %[[node]], %fs:0
 ; CHECK: calll _may_throw_or_crash
-; CHECK: movl 8(%esp), %[[next:[^ ,]*]]
+; CHECK: movl -16(%ebp), %[[next:[^ ,]*]]
 ; CHECK: movl %[[next]], %fs:0
 ; CHECK: retl
 
@@ -73,19 +79,34 @@ catchall:
 }
 
 ; CHECK-LABEL: _use_CxxFrameHandler3:
+; CHECK: pushl %ebp
+; CHECK: movl %esp, %ebp
 ; CHECK: subl ${{[0-9]+}}, %esp
-; CHECK: movl %esp, (%esp)
-; CHECK: movl $-1, 12(%esp)
-; CHECK: leal 4(%esp), %[[node:[^ ,]*]]
-; CHECK: movl $___ehhandler$use_CxxFrameHandler3, 8(%esp)
+; CHECK: movl %esp, -16(%ebp)
+; CHECK: movl $-1, -4(%ebp)
+; CHECK: leal -12(%ebp), %[[node:[^ ,]*]]
+; CHECK: movl $___ehhandler$use_CxxFrameHandler3, -8(%ebp)
 ; CHECK: movl %fs:0, %[[next:[^ ,]*]]
-; CHECK: movl %[[next]], 4(%esp)
+; CHECK: movl %[[next]], -12(%ebp)
 ; CHECK: movl %[[node]], %fs:0
+; CHECK: movl $0, -4(%ebp)
 ; CHECK: calll _may_throw_or_crash
-; CHECK: movl 4(%esp), %[[next:[^ ,]*]]
+; CHECK: movl -12(%ebp), %[[next:[^ ,]*]]
 ; CHECK: movl %[[next]], %fs:0
 ; CHECK: retl
 
+; CHECK: .section .xdata,"dr"
+; CHECK-LABEL: L__ehtable$use_CxxFrameHandler3:
+; CHECK-NEXT:  .long   429065506
+; CHECK-NEXT:  .long   2
+; CHECK-NEXT:  .long   ($stateUnwindMap$use_CxxFrameHandler3)
+; CHECK-NEXT:  .long   1
+; CHECK-NEXT:  .long   ($tryMap$use_CxxFrameHandler3)
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   0
+; CHECK-NEXT:  .long   1
+
 ; CHECK-LABEL: ___ehhandler$use_CxxFrameHandler3:
 ; CHECK: movl $L__ehtable$use_CxxFrameHandler3, %eax
 ; CHECK: jmp  ___CxxFrameHandler3 # TAILCALL
author	Dimitry Andric <dim@FreeBSD.org>	2015-06-09 19:06:30 +0000
committer	Dimitry Andric <dim@FreeBSD.org>	2015-06-09 19:06:30 +0000
commit	85d8b2bbe386bcfe669575d05b61482d7be07e5d (patch)
tree	1dc5e75ab222a9ead44c699eceafab7a6ca7b310 /test/CodeGen/X86
parent	5a5ac124e1efaf208671f01c46edb15f29ed2a0b (diff)