vendor/llvm/llvm-trunk-r290819

31 files changed, 1991 insertions, 68 deletions

diff --git a/test/CodeGen/SystemZ/asm-02.ll b/test/CodeGen/SystemZ/asm-02.ll
index 426d84882900..c916d827cde6 100644
--- a/test/CodeGen/SystemZ/asm-02.ll
+++ b/test/CodeGen/SystemZ/asm-02.ll
@@ -74,8 +74,8 @@ define void @f6(i64 %base, i64 %index) {
 ; Check that LAY is used if there is an index but the displacement is too large
 define void @f7(i64 %base, i64 %index) {
 ; CHECK-LABEL: f7:
-; CHECK: lay %r0, 4096(%r3,%r2)
-; CHECK: blah 0(%r0)
+; CHECK: lay %r1, 4096(%r3,%r2)
+; CHECK: blah 0(%r1)
 ; CHECK: br %r14
   %add = add i64 %base, 4096
   %addi = add i64 %add, %index
diff --git a/test/CodeGen/SystemZ/branch-07.ll b/test/CodeGen/SystemZ/branch-07.ll
index bac607133a89..857c9cb17adb 100644
--- a/test/CodeGen/SystemZ/branch-07.ll
+++ b/test/CodeGen/SystemZ/branch-07.ll
@@ -129,8 +129,8 @@ define void @f9(i64 %a, i64 %b, <2 x i64> *%dest) {
 ; CHECK-LABEL: f9:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: afi [[REG]], -268435456
-; CHECK: sllg [[REG2:%r[0-5]]], [[REG]], 32
-; CHECK: srag {{%r[0-5]}}, [[REG2]], 63
+; CHECK: risbg [[REG2:%r[0-5]]], [[REG]], 63, 191, 33
+; CHECK: lcgr  {{%r[0-5]}}, [[REG2]]
 ; CHECK: br %r14
   %avec = bitcast i64 %a to <2 x i32>
   %bvec = bitcast i64 %b to <2 x i32>
@@ -145,8 +145,8 @@ define void @f10(i64 %a, i64 %b, <2 x i64> *%dest) {
 ; CHECK-LABEL: f10:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: afi [[REG]], 1879048192
-; CHECK: sllg [[REG2:%r[0-5]]], [[REG]], 32
-; CHECK: srag {{%r[0-5]}}, [[REG2]], 63
+; CHECK: risbg [[REG2:%r[0-5]]], [[REG]], 63, 191, 33
+; CHECK: lcgr  {{%r[0-5]}}, [[REG2]]
 ; CHECK: br %r14
   %avec = bitcast i64 %a to <2 x i32>
   %bvec = bitcast i64 %b to <2 x i32>
diff --git a/test/CodeGen/SystemZ/cond-li.ll b/test/CodeGen/SystemZ/cond-li.ll
deleted file mode 100644
index a3e2f3fd1252..000000000000
--- a/test/CodeGen/SystemZ/cond-li.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; Test LOCHI/LOCGHI
-;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
-
-; CHECK-LABEL: bar1:
-; CHECK: lhi [[REG:%r[0-5]]], 42
-; CHECK: chi %r2, 0
-; CHECK: lochie [[REG]], 0
-define signext i32 @bar1(i32 signext %x) {
-  %cmp = icmp ne i32 %x, 0
-  %.x = select i1 %cmp, i32 42, i32 0
-  ret i32 %.x
-}
-
-; CHECK-LABEL: bar2:
-; CHECK: ltgr [[REG:%r[0-5]]], %r2
-; CHECK: lghi %r2, 42
-; CHECK: locghie %r2, 0
-define signext i64 @bar2(i64 signext %x) {
-  %cmp = icmp ne i64 %x, 0
-  %.x = select i1 %cmp, i64 42, i64 0
-  ret i64 %.x
-}
diff --git a/test/CodeGen/SystemZ/cond-load-01.ll b/test/CodeGen/SystemZ/cond-load-01.ll
index 97d4027126b8..c7ec4105cbc4 100644
--- a/test/CodeGen/SystemZ/cond-load-01.ll
+++ b/test/CodeGen/SystemZ/cond-load-01.ll
@@ -2,6 +2,10 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
+; Run the test again to make sure it still works the same even
+; in the presence of the load-store-on-condition-2 facility.
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
 declare i32 @foo(i32 *)
 
 ; Test the simple case.
@@ -128,3 +132,17 @@ exit:
   %res = phi i32 [ %easy, %entry ], [ %other, %load ]
   ret i32 %res
 }
+
+; Test that volatile loads do not use LOC, since if the condition is false,
+; it is unspecified whether or not the load happens.  LOCR is fine though.
+define i32 @f10(i32 %easy, i32 *%ptr, i32 %limit) {
+; CHECK-LABEL: f10:
+; CHECK: l {{%r[0-9]*}}, 0(%r3)
+; CHECK: locr
+; CHECK: br %r14
+  %cond = icmp ult i32 %limit, 42
+  %other = load volatile i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  ret i32 %res
+}
+
diff --git a/test/CodeGen/SystemZ/cond-load-02.ll b/test/CodeGen/SystemZ/cond-load-02.ll
index d0fe65e2fe0e..c40b2cca1de2 100644
--- a/test/CodeGen/SystemZ/cond-load-02.ll
+++ b/test/CodeGen/SystemZ/cond-load-02.ll
@@ -128,3 +128,17 @@ exit:
   %res = phi i64 [ %easy, %entry ], [ %other, %load ]
   ret i64 %res
 }
+
+; Test that volatile loads do not use LOCG, since if the condition is false,
+; it is unspecified whether or not the load happens.  LOCGR is fine though.
+define i64 @f10(i64 %easy, i64 *%ptr, i64 %limit) {
+; CHECK-LABEL: f10:
+; CHECK: lg {{%r[0-9]*}}, 0(%r3)
+; CHECK: locgr
+; CHECK: br %r14
+  %cond = icmp ult i64 %limit, 42
+  %other = load volatile i64, i64 *%ptr
+  %res = select i1 %cond, i64 %easy, i64 %other
+  ret i64 %res
+}
+
diff --git a/test/CodeGen/SystemZ/cond-load-03.ll b/test/CodeGen/SystemZ/cond-load-03.ll
new file mode 100644
index 000000000000..4cce92ea7188
--- /dev/null
+++ b/test/CodeGen/SystemZ/cond-load-03.ll
@@ -0,0 +1,159 @@
+; Test LOCFH.  See comments in asm-18.ll about testing high-word operations.
+;
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   -no-integrated-as | FileCheck %s
+
+declare void @foo(i32 *)
+
+; Test the simple case.
+define void @f1(i32 *%ptr, i32 %limit) {
+; CHECK-LABEL: f1:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: locfhhe [[REG]], 0(%r2)
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; ...and again with the operands swapped.
+define void @f2(i32 *%ptr, i32 %limit) {
+; CHECK-LABEL: f2:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: locfhl [[REG]], 0(%r2)
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %other, i32 %easy
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Check the high end of the aligned LOC range.
+define void @f3(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f3:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: locfhhe [[REG]], 524284(%r2)
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 131071
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Check the next word up.  Other sequences besides this one would be OK.
+define void @f4(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f4:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: agfi %r2, 524288
+; CHECK-DAG: clfi %r3, 42
+; CHECK: locfhhe [[REG]], 0(%r2)
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 131072
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Check the low end of the LOC range.
+define void @f5(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: locfhhe [[REG]], -524288(%r2)
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 -131072
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Check the next word down, with the same comments as f4.
+define void @f6(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f6:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK-DAG: agfi %r2, -524292
+; CHECK-DAG: clfi %r3, 42
+; CHECK: locfhhe [[REG]], 0(%r2)
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 -131073
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Try a frame index base.
+define void @f7(i32 %alt, i32 %limit) {
+; CHECK-LABEL: f7:
+; CHECK: brasl %r14, foo@PLT
+; CHECK: stepa [[REG:%r[0-5]]]
+; CHECK: locfhhe [[REG]], {{[0-9]+}}(%r15)
+; CHECK: br %r14
+  %ptr = alloca i32
+  call void @foo(i32 *%ptr)
+  %easy = call i32 asm "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Try a case when an index is involved.
+define void @f8(i32 %limit, i64 %base, i64 %index) {
+; CHECK-LABEL: f8:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r2, 42
+; CHECK: locfhhe [[REG]], 0({{%r[1-5]}})
+; CHECK: br %r14
+  %easy = call i32 asm "stepa $0", "=h"()
+  %add = add i64 %base, %index
+  %ptr = inttoptr i64 %add to i32 *
+  %cond = icmp ult i32 %limit, 42
+  %other = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %easy, i32 %other
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Test that conditionally-executed loads do not use LOC, since it is allowed
+; to trap even when the condition is false.
+define void @f9(i32 %limit, i32 *%ptr) {
+; CHECK-LABEL: f9:
+; CHECK-NOT: loc
+; CHECK: lfh
+; CHECK: br %r14
+entry:
+  %easy = call i32 asm "stepa $0", "=h"()
+  %cmp = icmp ule i32 %easy, %limit
+  br i1 %cmp, label %load, label %exit
+
+load:
+  %other = load i32, i32 *%ptr
+  br label %exit
+
+exit:
+  %res = phi i32 [ %easy, %entry ], [ %other, %load ]
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/cond-move-01.ll b/test/CodeGen/SystemZ/cond-move-01.ll
index 088dee0232ea..0be81c3ff801 100644
--- a/test/CodeGen/SystemZ/cond-move-01.ll
+++ b/test/CodeGen/SystemZ/cond-move-01.ll
@@ -1,6 +1,10 @@
 ; Test LOCR and LOCGR.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 -verify-machineinstrs | FileCheck %s
+;
+; Run the test again to make sure it still works the same even
+; in the presence of the load-store-on-condition-2 facility.
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -verify-machineinstrs | FileCheck %s
 
 ; Test LOCR.
 define i32 @f1(i32 %a, i32 %b, i32 %limit) {
@@ -46,3 +50,76 @@ define i64 @f4(i64 %a, i64 %b, i64 %limit) {
   %res = select i1 %cond, i64 %a, i64 %b
   ret i64 %res
 }
+
+; Check that we also get LOCR as a result of early if-conversion.
+define i32 @f5(i32 %a, i32 %b, i32 %limit) {
+; CHECK-LABEL: f5:
+; CHECK: clfi %r4, 41
+; CHECK: locrh %r2, %r3
+; CHECK: br %r14
+entry:
+  %cond = icmp ult i32 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ %a, %if.then ], [ %b, %entry ]
+  ret i32 %res
+}
+
+; ... and likewise for LOCGR.
+define i64 @f6(i64 %a, i64 %b, i64 %limit) {
+; CHECK-LABEL: f6:
+; CHECK: clgfi %r4, 41
+; CHECK: locgrh %r2, %r3
+; CHECK: br %r14
+entry:
+  %cond = icmp ult i64 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i64 [ %a, %if.then ], [ %b, %entry ]
+  ret i64 %res
+}
+
+; Check that inverting the condition works as well.
+define i32 @f7(i32 %a, i32 %b, i32 %limit) {
+; CHECK-LABEL: f7:
+; CHECK: clfi %r4, 41
+; CHECK: locrle %r2, %r3
+; CHECK: br %r14
+entry:
+  %cond = icmp ult i32 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ %b, %if.then ], [ %a, %entry ]
+  ret i32 %res
+}
+
+; ... and likewise for LOCGR.
+define i64 @f8(i64 %a, i64 %b, i64 %limit) {
+; CHECK-LABEL: f8:
+; CHECK: clgfi %r4, 41
+; CHECK: locgrle %r2, %r3
+; CHECK: br %r14
+entry:
+  %cond = icmp ult i64 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i64 [ %b, %if.then ], [ %a, %entry ]
+  ret i64 %res
+}
+
diff --git a/test/CodeGen/SystemZ/cond-move-02.ll b/test/CodeGen/SystemZ/cond-move-02.ll
new file mode 100644
index 000000000000..2e2bacd2532d
--- /dev/null
+++ b/test/CodeGen/SystemZ/cond-move-02.ll
@@ -0,0 +1,138 @@
+; Test LOCHI and LOCGHI.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -verify-machineinstrs | FileCheck %s
+
+define i32 @f1(i32 %x) {
+; CHECK-LABEL: f1:
+; CHECK: lhi [[REG:%r[0-5]]], 0
+; CHECK: chi %r2, 0
+; CHECK: lochilh [[REG]], 42
+; CHECK: br %r14
+  %cond = icmp ne i32 %x, 0
+  %res = select i1 %cond, i32 42, i32 0
+  ret i32 %res
+}
+
+define i32 @f2(i32 %x, i32 %y) {
+; CHECK-LABEL: f2:
+; CHECK: chi %r2, 0
+; CHECK: lochilh %r3, 42
+; CHECK: br %r14
+  %cond = icmp ne i32 %x, 0
+  %res = select i1 %cond, i32 42, i32 %y
+  ret i32 %res
+}
+
+define i32 @f3(i32 %x, i32 %y) {
+; CHECK-LABEL: f3:
+; CHECK: chi %r2, 0
+; CHECK: lochie %r3, 42
+; CHECK: br %r14
+  %cond = icmp ne i32 %x, 0
+  %res = select i1 %cond, i32 %y, i32 42
+  ret i32 %res
+}
+
+define i64 @f4(i64 %x) {
+; CHECK-LABEL: f4:
+; CHECK: lghi [[REG:%r[0-5]]], 0
+; CHECK: cghi %r2, 0
+; CHECK: locghilh [[REG]], 42
+; CHECK: br %r14
+  %cond = icmp ne i64 %x, 0
+  %res = select i1 %cond, i64 42, i64 0
+  ret i64 %res
+}
+
+define i64 @f5(i64 %x, i64 %y) {
+; CHECK-LABEL: f5:
+; CHECK: cghi %r2, 0
+; CHECK: locghilh %r3, 42
+; CHECK: br %r14
+  %cond = icmp ne i64 %x, 0
+  %res = select i1 %cond, i64 42, i64 %y
+  ret i64 %res
+}
+
+define i64 @f6(i64 %x, i64 %y) {
+; CHECK-LABEL: f6:
+; CHECK: cghi %r2, 0
+; CHECK: locghie %r3, 42
+; CHECK: br %r14
+  %cond = icmp ne i64 %x, 0
+  %res = select i1 %cond, i64 %y, i64 42
+  ret i64 %res
+}
+
+; Check that we also get LOCHI as a result of early if-conversion.
+define i32 @f7(i32 %x, i32 %y) {
+; CHECK-LABEL: f7:
+; CHECK: chi %r2, 0
+; CHECK: lochie %r3, 42
+; CHECK: br %r14
+entry:
+  %cond = icmp ne i32 %x, 0
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ %y, %if.then ], [ 42, %entry ]
+  ret i32 %res
+}
+
+; ... and the same for LOCGHI.
+define i64 @f8(i64 %x, i64 %y) {
+; CHECK-LABEL: f8:
+; CHECK: cghi %r2, 0
+; CHECK: locghie %r3, 42
+; CHECK: br %r14
+entry:
+  %cond = icmp ne i64 %x, 0
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i64 [ %y, %if.then ], [ 42, %entry ]
+  ret i64 %res
+}
+
+; Check that inverting the condition works as well.
+define i32 @f9(i32 %x, i32 %y) {
+; CHECK-LABEL: f9:
+; CHECK: chi %r2, 0
+; CHECK: lochilh %r3, 42
+; CHECK: br %r14
+entry:
+  %cond = icmp ne i32 %x, 0
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ 42, %if.then ], [ %y, %entry ]
+  ret i32 %res
+}
+
+; ... and the same for LOCGHI.
+define i64 @f10(i64 %x, i64 %y) {
+; CHECK-LABEL: f10:
+; CHECK: cghi %r2, 0
+; CHECK: locghilh %r3, 42
+; CHECK: br %r14
+entry:
+  %cond = icmp ne i64 %x, 0
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i64 [ 42, %if.then ], [ %y, %entry ]
+  ret i64 %res
+}
+
diff --git a/test/CodeGen/SystemZ/cond-move-03.ll b/test/CodeGen/SystemZ/cond-move-03.ll
new file mode 100644
index 000000000000..a9bf1c803109
--- /dev/null
+++ b/test/CodeGen/SystemZ/cond-move-03.ll
@@ -0,0 +1,213 @@
+; Test LOCFHR and LOCHHI.
+; See comments in asm-18.ll about testing high-word operations.
+;
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   -no-integrated-as | FileCheck %s
+
+define void @f1(i32 %limit) {
+; CHECK-LABEL: f1:
+; CHECK-DAG: stepa [[REG1:%r[0-5]]]
+; CHECK-DAG: stepb [[REG2:%r[0-5]]]
+; CHECK-DAG: clfi %r2, 42
+; CHECK: locfhrl [[REG2]], [[REG1]]
+; CHECK: stepc [[REG2]]
+; CHECK: br %r14
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %b = call i32 asm sideeffect "stepb $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 %a, i32 %b
+  call void asm sideeffect "stepc $0", "h"(i32 %res)
+  ret void
+}
+
+; FIXME: We should commute the LOCRMux to save one move.
+define void @f2(i32 %limit) {
+; CHECK-LABEL: f2:
+; CHECK-DAG: stepa [[REG1:%r[0-5]]]
+; CHECK-DAG: stepb [[REG2:%r[0-5]]]
+; CHECK-DAG: clijhe %r2, 42,
+; CHECK: risblg [[REG2]], [[REG1]], 0, 159, 32
+; CHECK: risbhg [[REG1]], [[REG2]], 0, 159, 32
+; CHECK: stepc [[REG1]]
+; CHECK: br %r14
+  %dummy = call i32 asm sideeffect "dummy $0", "=h"()
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %b = call i32 asm sideeffect "stepb $0", "=r"()
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 %a, i32 %b
+  call void asm sideeffect "stepc $0", "h"(i32 %res)
+  call void asm sideeffect "dummy $0", "h"(i32 %dummy)
+  ret void
+}
+
+define void @f3(i32 %limit) {
+; CHECK-LABEL: f3:
+; CHECK-DAG: stepa [[REG2:%r[0-5]]]
+; CHECK-DAG: stepb [[REG1:%r[0-5]]]
+; CHECK-DAG: clijhe %r2, 42,
+; CHECK: risbhg [[REG1]], [[REG2]], 0, 159, 32
+; CHECK: stepc [[REG1]]
+; CHECK: br %r14
+  %dummy = call i32 asm sideeffect "dummy $0", "=h"()
+  %a = call i32 asm sideeffect "stepa $0", "=r"()
+  %b = call i32 asm sideeffect "stepb $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 %a, i32 %b
+  call void asm sideeffect "stepc $0", "h"(i32 %res)
+  call void asm sideeffect "dummy $0", "h"(i32 %dummy)
+  ret void
+}
+
+; FIXME: We should commute the LOCRMux to save one move.
+define void @f4(i32 %limit) {
+; CHECK-LABEL: f4:
+; CHECK-DAG: stepa [[REG1:%r[0-5]]]
+; CHECK-DAG: stepb [[REG2:%r[0-5]]]
+; CHECK-DAG: clijhe %r2, 42,
+; CHECK: risbhg [[REG2]], [[REG1]], 0, 159, 32
+; CHECK: risblg [[REG1]], [[REG2]], 0, 159, 32
+; CHECK: stepc [[REG1]]
+; CHECK: br %r14
+  %dummy = call i32 asm sideeffect "dummy $0", "=h"()
+  %a = call i32 asm sideeffect "stepa $0", "=r"()
+  %b = call i32 asm sideeffect "stepb $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 %a, i32 %b
+  call void asm sideeffect "stepc $0", "r"(i32 %res)
+  call void asm sideeffect "dummy $0", "h"(i32 %dummy)
+  ret void
+}
+
+define void @f5(i32 %limit) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: stepa [[REG2:%r[0-5]]]
+; CHECK-DAG: stepb [[REG1:%r[0-5]]]
+; CHECK-DAG: clijhe %r2, 42,
+; CHECK: risblg [[REG1]], [[REG2]], 0, 159, 32
+; CHECK: stepc [[REG1]]
+; CHECK: br %r14
+  %dummy = call i32 asm sideeffect "dummy $0", "=h"()
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %b = call i32 asm sideeffect "stepb $0", "=r"()
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 %a, i32 %b
+  call void asm sideeffect "stepc $0", "r"(i32 %res)
+  call void asm sideeffect "dummy $0", "h"(i32 %dummy)
+  ret void
+}
+
+; Check that we also get LOCFHR as a result of early if-conversion.
+define void @f6(i32 %limit) {
+; CHECK-LABEL: f6:
+; CHECK-DAG: stepa [[REG1:%r[0-5]]]
+; CHECK-DAG: stepb [[REG2:%r[0-5]]]
+; CHECK-DAG: clfi %r2, 41
+; CHECK: locfhrle [[REG2]], [[REG1]]
+; CHECK: stepc [[REG2]]
+; CHECK: br %r14
+entry:
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %b = call i32 asm sideeffect "stepb $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ %a, %if.then ], [ %b, %entry ]
+  call void asm sideeffect "stepc $0", "h"(i32 %res)
+  ret void
+}
+
+; Check that inverting the condition works as well.
+define void @f7(i32 %limit) {
+; CHECK-LABEL: f7:
+; CHECK-DAG: stepa [[REG1:%r[0-5]]]
+; CHECK-DAG: stepb [[REG2:%r[0-5]]]
+; CHECK-DAG: clfi %r2, 41
+; CHECK: locfhrh [[REG2]], [[REG1]]
+; CHECK: stepc [[REG2]]
+; CHECK: br %r14
+entry:
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %b = call i32 asm sideeffect "stepb $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ %b, %if.then ], [ %a, %entry ]
+  call void asm sideeffect "stepc $0", "h"(i32 %res)
+  ret void
+}
+
+define void @f8(i32 %limit) {
+; CHECK-LABEL: f8:
+; CHECK: clfi %r2, 42
+; CHECK: lochhil [[REG:%r[0-5]]], 32767
+; CHECK: stepa [[REG]]
+; CHECK: br %r14
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 32767, i32 0
+  call void asm sideeffect "stepa $0", "h"(i32 %res)
+  ret void
+}
+
+define void @f9(i32 %limit) {
+; CHECK-LABEL: f9:
+; CHECK: clfi %r2, 42
+; CHECK: lochhil [[REG:%r[0-5]]], -32768
+; CHECK: stepa [[REG]]
+; CHECK: br %r14
+  %cond = icmp ult i32 %limit, 42
+  %res = select i1 %cond, i32 -32768, i32 0
+  call void asm sideeffect "stepa $0", "h"(i32 %res)
+  ret void
+}
+
+; Check that we also get LOCHHI as a result of early if-conversion.
+define void @f10(i32 %limit) {
+; CHECK-LABEL: f10:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r2, 41
+; CHECK: lochhile [[REG]], 123
+; CHECK: stepb [[REG]]
+; CHECK: br %r14
+entry:
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ 123, %if.then ], [ %a, %entry ]
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
+
+; Check that inverting the condition works as well.
+define void @f11(i32 %limit) {
+; CHECK-LABEL: f11:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r2, 41
+; CHECK: lochhih [[REG]], 123
+; CHECK: stepb [[REG]]
+; CHECK: br %r14
+entry:
+  %a = call i32 asm sideeffect "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  br i1 %cond, label %if.then, label %return
+
+if.then:
+  br label %return
+
+return:
+  %res = phi i32 [ %a, %if.then ], [ 123, %entry ]
+  call void asm sideeffect "stepb $0", "h"(i32 %res)
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/cond-store-07.ll b/test/CodeGen/SystemZ/cond-store-07.ll
index 35b1303f4024..79b4f87006b5 100644
--- a/test/CodeGen/SystemZ/cond-store-07.ll
+++ b/test/CodeGen/SystemZ/cond-store-07.ll
@@ -2,6 +2,10 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
+; Run the test again to make sure it still works the same even
+; in the presence of the load-store-on-condition-2 facility.
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
 declare void @foo(i32 *)
 
 ; Test the simple case, with the loaded value first.
diff --git a/test/CodeGen/SystemZ/cond-store-09.ll b/test/CodeGen/SystemZ/cond-store-09.ll
new file mode 100644
index 000000000000..bf7a8b880075
--- /dev/null
+++ b/test/CodeGen/SystemZ/cond-store-09.ll
@@ -0,0 +1,142 @@
+; Test STOCFHs that are presented as selects.
+; See comments in asm-18.ll about testing high-word operations.
+;
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:   -no-integrated-as | FileCheck %s
+
+declare void @foo(i32 *)
+
+; Test the simple case, with the loaded value first.
+define void @f1(i32 *%ptr, i32 %limit) {
+; CHECK-LABEL: f1:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: stocfhhe [[REG]], 0(%r2)
+; CHECK: br %r14
+  %alt = call i32 asm "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %orig, i32 %alt
+  store i32 %res, i32 *%ptr
+  ret void
+}
+
+; ...and with the loaded value second
+define void @f2(i32 *%ptr, i32 %limit) {
+; CHECK-LABEL: f2:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: stocfhl [[REG]], 0(%r2)
+; CHECK: br %r14
+  %alt = call i32 asm "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %alt, i32 %orig
+  store i32 %res, i32 *%ptr
+  ret void
+}
+
+; Check the high end of the aligned STOC range.
+define void @f3(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f3:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: stocfhhe [[REG]], 524284(%r2)
+; CHECK: br %r14
+  %alt = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 131071
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %orig, i32 %alt
+  store i32 %res, i32 *%ptr
+  ret void
+}
+
+; Check the next word up.  Other sequences besides this one would be OK.
+define void @f4(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f4:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: agfi %r2, 524288
+; CHECK-DAG: clfi %r3, 42
+; CHECK: stocfhhe [[REG]], 0(%r2)
+; CHECK: br %r14
+  %alt = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 131072
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %orig, i32 %alt
+  store i32 %res, i32 *%ptr
+  ret void
+}
+
+; Check the low end of the STOC range.
+define void @f5(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f5:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: clfi %r3, 42
+; CHECK: stocfhhe [[REG]], -524288(%r2)
+; CHECK: br %r14
+  %alt = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 -131072
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %orig, i32 %alt
+  store i32 %res, i32 *%ptr
+  ret void
+}
+
+; Check the next word down, with the same comments as f8.
+define void @f6(i32 *%base, i32 %limit) {
+; CHECK-LABEL: f6:
+; CHECK-DAG: stepa [[REG:%r[0-5]]]
+; CHECK-DAG: agfi %r2, -524292
+; CHECK-DAG: clfi %r3, 42
+; CHECK: stocfhhe [[REG]], 0(%r2)
+; CHECK: br %r14
+  %alt = call i32 asm "stepa $0", "=h"()
+  %ptr = getelementptr i32, i32 *%base, i64 -131073
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %orig, i32 %alt
+  store i32 %res, i32 *%ptr
+  ret void
+}
+
+; Try a frame index base.
+define void @f7(i32 %limit) {
+; CHECK-LABEL: f7:
+; CHECK: brasl %r14, foo@PLT
+; CHECK: stepa [[REG:%r[0-5]]]
+; CHECK: stocfhhe [[REG]], {{[0-9]+}}(%r15)
+; CHECK: brasl %r14, foo@PLT
+; CHECK: br %r14
+  %ptr = alloca i32
+  call void @foo(i32 *%ptr)
+  %alt = call i32 asm "stepa $0", "=h"()
+  %cond = icmp ult i32 %limit, 42
+  %orig = load i32, i32 *%ptr
+  %res = select i1 %cond, i32 %orig, i32 %alt
+  store i32 %res, i32 *%ptr
+  call void @foo(i32 *%ptr)
+  ret void
+}
+
+; Test that conditionally-executed stores do not use STOC, since STOC
+; is allowed to trap even when the condition is false.
+define void @f8(i32 %a, i32 %b, i32 *%dest) {
+; CHECK-LABEL: f8:
+; CHECK-NOT: stoc
+; CHECK: stfh
+; CHECK: br %r14
+entry:
+  %val = call i32 asm "stepa $0", "=h"()
+  %cmp = icmp ule i32 %a, %b
+  br i1 %cmp, label %store, label %exit
+
+store:
+  store i32 %val, i32 *%dest
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/test/CodeGen/SystemZ/fp-const-10.ll b/test/CodeGen/SystemZ/fp-const-10.ll
new file mode 100644
index 000000000000..cb00eb64a0d0
--- /dev/null
+++ b/test/CodeGen/SystemZ/fp-const-10.ll
@@ -0,0 +1,15 @@
+; Test loads of SNaN.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+; Test that we don't do an FP extending load, as this would result in a
+; converstion to QNaN.
+define double @f1() {
+; CHECK-LABEL: .LCPI0_0
+; CHECK:      .quad   9219994337134247936
+; CHECK-LABEL: f1:
+; CHECK:      larl    %r1, .LCPI0_0
+; CHECK-NOT:  ldeb    %f0, 0(%r1)
+; CHECK:      ld      %f0, 0(%r1)
+  ret double 0x7FF4000000000000
+}
diff --git a/test/CodeGen/SystemZ/fpc-intrinsics.ll b/test/CodeGen/SystemZ/fpc-intrinsics.ll
new file mode 100644
index 000000000000..fd8064b18fcf
--- /dev/null
+++ b/test/CodeGen/SystemZ/fpc-intrinsics.ll
@@ -0,0 +1,67 @@
+; Test floating-point control register intrinsics.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare void @llvm.s390.sfpc(i32)
+declare i32 @llvm.s390.efpc()
+
+; SFPC.
+define void @test_sfpc(i32 %fpc) {
+; CHECK-LABEL: test_sfpc:
+; CHECK: sfpc %r2
+; CHECK: br %r14
+  call void @llvm.s390.sfpc(i32 %fpc)
+  ret void
+}
+
+; EFPC.
+define i32 @test_efpc() {
+; CHECK-LABEL: test_efpc:
+; CHECK: efpc %r2
+; CHECK: br %r14
+  %res = call i32 @llvm.s390.efpc()
+  ret i32 %res
+}
+
+; LFPC.
+define void @test_lfpc1(i32 *%ptr) {
+; CHECK-LABEL: test_lfpc1:
+; CHECK: lfpc 0(%r2)
+; CHECK: br %r14
+  %fpc = load i32, i32 *%ptr
+  call void @llvm.s390.sfpc(i32 %fpc)
+  ret void
+}
+
+; LFPC with offset.
+define void @test_lfpc2(i32 *%ptr) {
+; CHECK-LABEL: test_lfpc2:
+; CHECK: lfpc 4092(%r2)
+; CHECK: br %r14
+  %ptr1 = getelementptr i32, i32 *%ptr, i32 1023
+  %fpc = load i32, i32 *%ptr1
+  call void @llvm.s390.sfpc(i32 %fpc)
+  ret void
+}
+
+; STFPC.
+define void @test_stfpc1(i32 *%ptr) {
+; CHECK-LABEL: test_stfpc1:
+; CHECK: stfpc 0(%r2)
+; CHECK: br %r14
+  %fpc = call i32 @llvm.s390.efpc()
+  store i32 %fpc, i32 *%ptr
+  ret void
+}
+
+; STFPC with offset.
+define void @test_stfpc2(i32 *%ptr) {
+; CHECK-LABEL: test_stfpc2:
+; CHECK: stfpc 4092(%r2)
+; CHECK: br %r14
+  %fpc = call i32 @llvm.s390.efpc()
+  %ptr1 = getelementptr i32, i32 *%ptr, i32 1023
+  store i32 %fpc, i32 *%ptr1
+  ret void
+}
+
diff --git a/test/CodeGen/SystemZ/int-cmp-48.ll b/test/CodeGen/SystemZ/int-cmp-48.ll
index 277423b8cc0c..2a6d9d5fcaf0 100644
--- a/test/CodeGen/SystemZ/int-cmp-48.ll
+++ b/test/CodeGen/SystemZ/int-cmp-48.ll
@@ -29,8 +29,8 @@ exit:
 define void @f2(i8 *%src) {
 ; CHECK-LABEL: f2:
 ; CHECK: llc [[REG:%r[0-5]]], 0(%r2)
-; CHECK: mvi 0(%r2), 0
 ; CHECK: tmll [[REG]], 1
+; CHECK: mvi 0(%r2), 0
 ; CHECK: ber %r14
 ; CHECK: br %r14
 entry:
diff --git a/test/CodeGen/SystemZ/int-conv-12.ll b/test/CodeGen/SystemZ/int-conv-12.ll
new file mode 100644
index 000000000000..bedd295782e9
--- /dev/null
+++ b/test/CodeGen/SystemZ/int-conv-12.ll
@@ -0,0 +1,133 @@
+; Test 31-to-64 bit zero extensions.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+; Test register extension, starting with an i64.
+define i64 @f1(i64 %a) {
+; CHECK-LABEL: f1:
+; CHECK: llgtr %r2, %r2
+; CHECK: br %r14
+  %ext = and i64 %a, 2147483647
+  ret i64 %ext
+}
+
+; Test register extension, starting with an i32.
+define i64 @f2(i32 %a) {
+; CHECK-LABEL: f2:
+; CHECK: llgtr %r2, %r2
+; CHECK: br %r14
+  %and = and i32 %a, 2147483647
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; ... and the other way around.
+define i64 @f3(i32 %a) {
+; CHECK-LABEL: f3:
+; CHECK: llgtr %r2, %r2
+; CHECK: br %r14
+  %ext = zext i32 %a to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; Check LLGT with no displacement.
+define i64 @f4(i32 *%src) {
+; CHECK-LABEL: f4:
+; CHECK: llgt %r2, 0(%r2)
+; CHECK: br %r14
+  %word = load i32, i32 *%src
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; ... and the other way around.
+define i64 @f5(i32 *%src) {
+; CHECK-LABEL: f5:
+; CHECK: llgt %r2, 0(%r2)
+; CHECK: br %r14
+  %word = load i32, i32 *%src
+  %and = and i32 %word, 2147483647
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check the high end of the LLGT range.
+define i64 @f6(i32 *%src) {
+; CHECK-LABEL: f6:
+; CHECK: llgt %r2, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %word = load i32 , i32 *%ptr
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f7(i32 *%src) {
+; CHECK-LABEL: f7:
+; CHECK: agfi %r2, 524288
+; CHECK: llgt %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %word = load i32 , i32 *%ptr
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; Check the high end of the negative LLGT range.
+define i64 @f8(i32 *%src) {
+; CHECK-LABEL: f8:
+; CHECK: llgt %r2, -4(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -1
+  %word = load i32 , i32 *%ptr
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; Check the low end of the LLGT range.
+define i64 @f9(i32 *%src) {
+; CHECK-LABEL: f9:
+; CHECK: llgt %r2, -524288(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -131072
+  %word = load i32 , i32 *%ptr
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; Check the next word down, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f10(i32 *%src) {
+; CHECK-LABEL: f10:
+; CHECK: agfi %r2, -524292
+; CHECK: llgt %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -131073
+  %word = load i32 , i32 *%ptr
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
+; Check that LLGT allows an index.
+define i64 @f11(i64 %src, i64 %index) {
+; CHECK-LABEL: f11:
+; CHECK: llgt %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %word = load i32 , i32 *%ptr
+  %ext = zext i32 %word to i64
+  %and = and i64 %ext, 2147483647
+  ret i64 %and
+}
+
diff --git a/test/CodeGen/SystemZ/int-conv-13.ll b/test/CodeGen/SystemZ/int-conv-13.ll
new file mode 100644
index 000000000000..4c25447c4a94
--- /dev/null
+++ b/test/CodeGen/SystemZ/int-conv-13.ll
@@ -0,0 +1,278 @@
+; Test load and zero rightmost byte.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+; Check LZRF with no displacement.
+define i32 @f1(i32 *%src) {
+; CHECK-LABEL: f1:
+; CHECK: lzrf %r2, 0(%r2)
+; CHECK: br %r14
+  %val = load i32, i32 *%src
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check the high end of the LZRF range.
+define i32 @f2(i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: lzrf %r2, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i32 @f3(i32 *%src) {
+; CHECK-LABEL: f3:
+; CHECK: agfi %r2, 524288
+; CHECK: lzrf %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check the high end of the negative LZRF range.
+define i32 @f4(i32 *%src) {
+; CHECK-LABEL: f4:
+; CHECK: lzrf %r2, -4(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -1
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check the low end of the LZRF range.
+define i32 @f5(i32 *%src) {
+; CHECK-LABEL: f5:
+; CHECK: lzrf %r2, -524288(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -131072
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check the next word down, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i32 @f6(i32 *%src) {
+; CHECK-LABEL: f6:
+; CHECK: agfi %r2, -524292
+; CHECK: lzrf %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -131073
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check that LZRF allows an index.
+define i32 @f7(i64 %src, i64 %index) {
+; CHECK-LABEL: f7:
+; CHECK: lzrf %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %val = load i32 , i32 *%ptr
+  %and = and i32 %val, 4294967040
+  ret i32 %and
+}
+
+; Check LZRG with no displacement.
+define i64 @f8(i64 *%src) {
+; CHECK-LABEL: f8:
+; CHECK: lzrg %r2, 0(%r2)
+; CHECK: br %r14
+  %val = load i64, i64 *%src
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check the high end of the LZRG range.
+define i64 @f9(i64 *%src) {
+; CHECK-LABEL: f9:
+; CHECK: lzrg %r2, 524280(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 65535
+  %val = load i64, i64 *%ptr
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f10(i64 *%src) {
+; CHECK-LABEL: f10:
+; CHECK: agfi %r2, 524288
+; CHECK: lzrg %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 65536
+  %val = load i64, i64 *%ptr
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check the high end of the negative LZRG range.
+define i64 @f11(i64 *%src) {
+; CHECK-LABEL: f11:
+; CHECK: lzrg %r2, -8(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 -1
+  %val = load i64, i64 *%ptr
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check the low end of the LZRG range.
+define i64 @f12(i64 *%src) {
+; CHECK-LABEL: f12:
+; CHECK: lzrg %r2, -524288(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 -65536
+  %val = load i64, i64 *%ptr
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check the next word down, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f13(i64 *%src) {
+; CHECK-LABEL: f13:
+; CHECK: agfi %r2, -524296
+; CHECK: lzrg %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 -65537
+  %val = load i64, i64 *%ptr
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check that LZRG allows an index.
+define i64 @f14(i64 %src, i64 %index) {
+; CHECK-LABEL: f14:
+; CHECK: lzrg %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i64 *
+  %val = load i64 , i64 *%ptr
+  %and = and i64 %val, 18446744073709551360
+  ret i64 %and
+}
+
+; Check LLZRGF with no displacement.
+define i64 @f15(i32 *%src) {
+; CHECK-LABEL: f15:
+; CHECK: llzrgf %r2, 0(%r2)
+; CHECK: br %r14
+  %val = load i32, i32 *%src
+  %ext = zext i32 %val to i64
+  %and = and i64 %ext, 18446744073709551360
+  ret i64 %and
+}
+
+; ... and the other way around.
+define i64 @f16(i32 *%src) {
+; CHECK-LABEL: f16:
+; CHECK: llzrgf %r2, 0(%r2)
+; CHECK: br %r14
+  %val = load i32, i32 *%src
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check the high end of the LLZRGF range.
+define i64 @f17(i32 *%src) {
+; CHECK-LABEL: f17:
+; CHECK: llzrgf %r2, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f18(i32 *%src) {
+; CHECK-LABEL: f18:
+; CHECK: agfi %r2, 524288
+; CHECK: llzrgf %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check the high end of the negative LLZRGF range.
+define i64 @f19(i32 *%src) {
+; CHECK-LABEL: f19:
+; CHECK: llzrgf %r2, -4(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -1
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check the low end of the LLZRGF range.
+define i64 @f20(i32 *%src) {
+; CHECK-LABEL: f20:
+; CHECK: llzrgf %r2, -524288(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -131072
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check the next word down, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f21(i32 *%src) {
+; CHECK-LABEL: f21:
+; CHECK: agfi %r2, -524292
+; CHECK: llzrgf %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 -131073
+  %val = load i32, i32 *%ptr
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check that LLZRGF allows an index.
+define i64 @f22(i64 %src, i64 %index) {
+; CHECK-LABEL: f22:
+; CHECK: llzrgf %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %val = load i32 , i32 *%ptr
+  %and = and i32 %val, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
+; Check that we still get a RISBGN if the source is in a register.
+define i64 @f23(i32 %src) {
+; CHECK-LABEL: f23:
+; CHECK: risbgn %r2, %r2, 32, 183, 0
+; CHECK: br %r14
+  %and = and i32 %src, 4294967040
+  %ext = zext i32 %and to i64
+  ret i64 %ext
+}
+
diff --git a/test/CodeGen/SystemZ/loop-01.ll b/test/CodeGen/SystemZ/loop-01.ll
index b51c96d52e3a..321be4b8e62b 100644
--- a/test/CodeGen/SystemZ/loop-01.ll
+++ b/test/CodeGen/SystemZ/loop-01.ll
@@ -1,6 +1,8 @@
 ; Test loop tuning.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
+; RUN:  | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-Z13
 
 ; Test that strength reduction is applied to addresses with a scale factor,
 ; but that indexed addressing can still be used.
@@ -122,3 +124,118 @@ loop.next:
 exit:
   ret void
 }
+
+; Test that negative offsets are avoided for loads of floating point.
+%s.float = type { float, float, float }
+define void @f5(%s.float* nocapture %a,
+                %s.float* nocapture readonly %b,
+                i32 zeroext %S) {
+; CHECK-Z13-LABEL: f5:
+; CHECK-Z13-NOT: -{{[0-9]+}}(%r
+
+entry:
+  %cmp9 = icmp eq i32 %S, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                 ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:          ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                   ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                           ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a1 = getelementptr inbounds %s.float, %s.float* %b, i64 %indvars.iv, i32 0
+  %tmp = load float, float* %a1, align 4
+  %b4 = getelementptr inbounds %s.float, %s.float* %b, i64 %indvars.iv, i32 1
+  %tmp1 = load float, float* %b4, align 4
+  %add = fadd float %tmp, %tmp1
+  %c = getelementptr inbounds %s.float, %s.float* %b, i64 %indvars.iv, i32 2
+  %tmp2 = load float, float* %c, align 4
+  %add7 = fadd float %add, %tmp2
+  %a10 = getelementptr inbounds %s.float, %s.float* %a, i64 %indvars.iv, i32 0
+  store float %add7, float* %a10, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %S
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+; Test that negative offsets are avoided for loads of double.
+%s.double = type { double, double, double }
+define void @f6(%s.double* nocapture %a,
+                %s.double* nocapture readonly %b,
+                i32 zeroext %S) {
+; CHECK-Z13-LABEL: f6:
+; CHECK-Z13-NOT: -{{[0-9]+}}(%r
+entry:
+  %cmp9 = icmp eq i32 %S, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                  ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:           ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                    ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                            ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a1 = getelementptr inbounds %s.double, %s.double* %b, i64 %indvars.iv, i32 0
+  %tmp = load double, double* %a1, align 4
+  %b4 = getelementptr inbounds %s.double, %s.double* %b, i64 %indvars.iv, i32 1
+  %tmp1 = load double, double* %b4, align 4
+  %add = fadd double %tmp, %tmp1
+  %c = getelementptr inbounds %s.double, %s.double* %b, i64 %indvars.iv, i32 2
+  %tmp2 = load double, double* %c, align 4
+  %add7 = fadd double %add, %tmp2
+  %a10 = getelementptr inbounds %s.double, %s.double* %a, i64 %indvars.iv, i32 0
+  store double %add7, double* %a10, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %S
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+; Test that negative offsets are avoided for memory accesses of vector type.
+%s.vec = type { <4 x i32>, <4 x i32>, <4 x i32> }
+define void @f7(%s.vec* nocapture %a,
+                %s.vec* nocapture readonly %b,
+                i32 zeroext %S) {
+; CHECK-Z13-LABEL: f7:
+; CHECK-Z13-NOT: -{{[0-9]+}}(%r
+entry:
+  %cmp9 = icmp eq i32 %S, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                 ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:          ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                   ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                           ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a1 = getelementptr inbounds %s.vec, %s.vec* %b, i64 %indvars.iv, i32 0
+  %tmp = load <4 x i32>, <4 x i32>* %a1, align 4
+  %b4 = getelementptr inbounds %s.vec, %s.vec* %b, i64 %indvars.iv, i32 1
+  %tmp1 = load <4 x i32>, <4 x i32>* %b4, align 4
+  %add = add <4 x i32> %tmp1, %tmp
+  %c = getelementptr inbounds %s.vec, %s.vec* %b, i64 %indvars.iv, i32 2
+  %tmp2 = load <4 x i32>, <4 x i32>* %c, align 4
+  %add7 = add <4 x i32> %add, %tmp2
+  %a10 = getelementptr inbounds %s.vec, %s.vec* %a, i64 %indvars.iv, i32 0
+  store <4 x i32> %add7, <4 x i32>* %a10, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %S
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/test/CodeGen/SystemZ/loop-02.ll b/test/CodeGen/SystemZ/loop-02.ll
new file mode 100644
index 000000000000..a3c1cf65ae27
--- /dev/null
+++ b/test/CodeGen/SystemZ/loop-02.ll
@@ -0,0 +1,38 @@
+; Test BRCTH.
+
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z196 \
+; RUN:   -no-integrated-as | FileCheck %s
+
+; Test a loop that should be converted into dbr form and then use BRCTH.
+define void @f2(i32 *%src, i32 *%dest) {
+; CHECK-LABEL: f2:
+; CHECK: blah [[REG:%r[0-5]]]
+; CHECK: [[LABEL:\.[^:]*]]:{{.*}} %loop
+; CHECK: brcth [[REG]], [[LABEL]]
+; CHECK: br %r14
+entry:
+  ; Force upper bound into a high register in order to encourage the
+  ; register allocator to use a high register for the count variable.
+  %top = call i32 asm sideeffect "blah $0", "=h"()
+  br label %loop
+
+loop:
+  %count = phi i32 [ 0, %entry ], [ %next, %loop.next ]
+  %next = add i32 %count, 1
+  %val = load volatile i32 , i32 *%src
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %loop.next, label %loop.store
+
+loop.store:
+  %add = add i32 %val, 1
+  store volatile i32 %add, i32 *%dest
+  br label %loop.next
+
+loop.next:
+  %cont = icmp ne i32 %next, %top
+  br i1 %cont, label %loop, label %exit
+
+exit:
+  ret void
+}
+
diff --git a/test/CodeGen/SystemZ/risbg-01.ll b/test/CodeGen/SystemZ/risbg-01.ll
index 1c4315343de0..6a146fab82c6 100644
--- a/test/CodeGen/SystemZ/risbg-01.ll
+++ b/test/CodeGen/SystemZ/risbg-01.ll
@@ -472,9 +472,9 @@ define i64 @f41(i32 %a) {
 ; when testing whether the shifted-in bits of the shift right were significant.
 define i64 @f42(i1 %x) {
 ; CHECK-LABEL: f42:
-; CHECK: sll %r2, 31
-; CHECK: sra %r2, 31
-; CHECK: llgcr %r2, %r2
+; CHECK: nilf  %r2, 1
+; CHECK: lcr %r0, %r2
+; CHECK: llgcr %r2, %r0
 ; CHECK: br %r14
   %ext = sext i1 %x to i8
   %ext2 = zext i8 %ext to i64
diff --git a/test/CodeGen/SystemZ/shift-10.ll b/test/CodeGen/SystemZ/shift-10.ll
index bf2f0f1776ee..176f7dea8c35 100644
--- a/test/CodeGen/SystemZ/shift-10.ll
+++ b/test/CodeGen/SystemZ/shift-10.ll
@@ -5,8 +5,8 @@
 ; Test a shift right followed by a sign extension.  This can use two shifts.
 define i64 @f1(i32 %a) {
 ; CHECK-LABEL: f1:
-; CHECK: sllg [[REG:%r[0-5]]], %r2, 62
-; CHECK: srag %r2, [[REG]], 63
+; CHECK: risbg %r0, %r2, 63, 191, 63
+; CHECK: lcgr  %r2, %r0
 ; CHECK: br %r14
   %shr = lshr i32 %a, 1
   %trunc = trunc i32 %shr to i1
@@ -18,8 +18,8 @@ define i64 @f1(i32 %a) {
 ; ashr/sext pair.
 define i64 @f2(i32 %a) {
 ; CHECK-LABEL: f2:
-; CHECK: sllg [[REG:%r[0-5]]], %r2, 33
-; CHECK: srag %r2, [[REG]], 63
+; CHECK: risbg %r0, %r2, 63, 191, 34
+; CHECK: lcgr  %r2, %r0
 ; CHECK: br %r14
   %shr = lshr i32 %a, 30
   %trunc = trunc i32 %shr to i1
diff --git a/test/CodeGen/SystemZ/shift-11.ll b/test/CodeGen/SystemZ/shift-11.ll
index 9741fa5a0b55..3d6fc6a05143 100644
--- a/test/CodeGen/SystemZ/shift-11.ll
+++ b/test/CodeGen/SystemZ/shift-11.ll
@@ -61,3 +61,25 @@ define i64 @f6(i64 %a, i64 %sh) {
   %shift = shl i64 %a, %and
   ret i64 %shift
 }
+
+; Test shift with negative 32-bit value.
+define i32 @f8(i32 %a, i32 %sh, i32 %test) {
+; CHECK-LABEL: f8:
+; CHECK: nill %r3, 65529
+; CHECK: sll %r2, 0(%r3)
+  %and = and i32 %sh, -7
+  %shift = shl i32 %a, %and
+
+  ret i32 %shift
+}
+
+; Test shift with negative 64-bit value.
+define i64 @f9(i64 %a, i64 %sh, i64 %test) {
+; CHECK-LABEL: f9:
+; CHECK: nill %r3, 65529
+; CHECK: sllg %r2, %r2, 0(%r3)
+  %and = and i64 %sh, -7
+  %shift = shl i64 %a, %and
+
+  ret i64 %shift
+}
diff --git a/test/CodeGen/SystemZ/swift-return.ll b/test/CodeGen/SystemZ/swift-return.ll
index e72d6def84e8..69d0e979190c 100644
--- a/test/CodeGen/SystemZ/swift-return.ll
+++ b/test/CodeGen/SystemZ/swift-return.ll
@@ -49,10 +49,9 @@ declare swiftcc { i16, i8 } @gen(i32)
 ; CHECK: a %r2, 172(%r15)
 ; CHECK: a %r2, 176(%r15)
 ; CHECK-O0-LABEL: test2:
-; CHECK-O0: la %[[REG1:r[0-9]+]], 168(%r15)
 ; CHECK-O0: st %r2, [[SPILL1:[0-9]+]](%r15)
-; CHECK-O0: lgr %r2, %[[REG1]]
 ; CHECK-O0: l %r3, [[SPILL1]](%r15)
+; CHECK-O0: la %r2, 168(%r15)
 ; CHECK-O0: brasl %r14, gen2
 ; CHECK-O0-DAG: l %r{{.*}}, 184(%r15)
 ; CHECK-O0-DAG: l %r{{.*}}, 180(%r15)
diff --git a/test/CodeGen/SystemZ/swifterror.ll b/test/CodeGen/SystemZ/swifterror.ll
index 90d55eef4aef..092a7b133f13 100644
--- a/test/CodeGen/SystemZ/swifterror.ll
+++ b/test/CodeGen/SystemZ/swifterror.ll
@@ -16,9 +16,8 @@ define float @foo(%swift_error** swifterror %error_ptr_ref) {
 ; CHECK-O0-LABEL: foo:
 ; CHECK-O0: lghi %r2, 16
 ; CHECK-O0: brasl %r14, malloc
-; CHECK-O0: lgr %r[[REG1:[0-9]+]], %r2
+; CHECK-O0: lgr %r9, %r2
 ; CHECK-O0: mvi 8(%r2), 1
-; CHECK-O0: lgr %r9, %r[[REG1]]
 entry:
   %call = call i8* @malloc(i64 16)
   %call.0 = bitcast i8* %call to %swift_error*
@@ -130,7 +129,8 @@ define float @foo_if(%swift_error** swifterror %error_ptr_ref, i32 %cc) {
 ; CHECK-O0: lgr %r9, %r[[REG1]]
 ; CHECK-O0: br %r14
 ; reload from stack
-; CHECK-O0: lg %r9, [[OFFS]](%r15)
+; CHECK-O0: lg %r[[REG2:[0-9]+]], [[OFFS]](%r15)
+; CHECK-O0: lgr %r9, %r[[REG2]]
 ; CHECK-O0: br %r14
 entry:
   %cond = icmp ne i32 %cc, 0
@@ -172,7 +172,8 @@ define float @foo_loop(%swift_error** swifterror %error_ptr_ref, i32 %cc, float
 ; CHECK-O0: mvi 8(%r2), 1
 ; CHECK-O0: jnh
 ; reload from stack
-; CHECK-O0: lg %r9, [[OFFS:[0-9]+]](%r15)
+; CHECK-O0: lg %r[[REG2:[0-9]+]], [[OFFS:[0-9]+]](%r15)
+; CHECK-O0: lgr %r9, %r[[REG2]]
 ; CHECK-O0: br %r14
 entry:
   br label %bb_loop
diff --git a/test/CodeGen/SystemZ/tdc-06.ll b/test/CodeGen/SystemZ/tdc-06.ll
index 11fb1e2916e0..a099c00d227b 100644
--- a/test/CodeGen/SystemZ/tdc-06.ll
+++ b/test/CodeGen/SystemZ/tdc-06.ll
@@ -14,14 +14,14 @@ entry:
 ; CHECK: ltdbr %f0, %f0
 ; CHECK: je [[RET:.L.*]]
   %testeq = fcmp oeq double %x, 0.000000e+00
-  br i1 %testeq, label %ret, label %nonzero
+  br i1 %testeq, label %ret, label %nonzero, !prof !1
 
 nonzero:
 ; CHECK: lhi %r2, 1
 ; CHECK: cdbr %f0, %f0
 ; CHECK: jo [[RET]]
   %testnan = fcmp uno double %x, 0.000000e+00
-  br i1 %testnan, label %ret, label %nonzeroord
+  br i1 %testnan, label %ret, label %nonzeroord, !prof !1
 
 nonzeroord:
 ; CHECK: lhi %r2, 2
@@ -29,7 +29,7 @@ nonzeroord:
 ; CHECK: jl [[RET]]
   %abs = tail call double @llvm.fabs.f64(double %x)
   %testinf = fcmp oeq double %abs, 0x7FF0000000000000
-  br i1 %testinf, label %ret, label %finite
+  br i1 %testinf, label %ret, label %finite, !prof !1
 
 finite:
 ; CHECK: lhi %r2, 3
@@ -46,3 +46,5 @@ ret:
   %res = phi i32 [ 5, %entry ], [ 1, %nonzero ], [ 2, %nonzeroord ], [ %finres, %finite ]
   ret i32 %res
 }
+
+!1 = !{!"branch_weights", i32 1, i32 1}
diff --git a/test/CodeGen/SystemZ/trap-02.ll b/test/CodeGen/SystemZ/trap-02.ll
new file mode 100644
index 000000000000..5fdb87c292d7
--- /dev/null
+++ b/test/CodeGen/SystemZ/trap-02.ll
@@ -0,0 +1,90 @@
+; Test zE12 conditional traps
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 | FileCheck %s
+
+declare void @llvm.trap()
+
+; Check conditional compare logical and trap
+define i32 @f1(i32 zeroext %a, i32 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: clth %r2, 0(%r3)
+; CHECK: lhi %r2, 0
+; CHECK: br %r14
+entry:
+  %b = load i32, i32 *%ptr
+  %cmp = icmp ugt i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 0
+}
+
+; Check conditional compare logical grande and trap
+define i64 @f2(i64 zeroext %a, i64 *%ptr) {
+; CHECK-LABEL: f2:
+; CHECK: clgtl %r2, 0(%r3)
+; CHECK: lghi %r2, 0
+; CHECK: br %r14
+entry:
+  %b = load i64, i64 *%ptr
+  %cmp = icmp ult i64 %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 0
+}
+
+; Verify that we don't attempt to use the compare and trap
+; instruction with an index operand.
+define i32 @f3(i32 zeroext %a, i32 *%base, i64 %offset) {
+; CHECK-LABEL: f3:
+; CHECK: cl %r2, 0(%r{{[0-5]}},%r3)
+; CHECK-LABEL: .Ltmp0
+; CHECK: jh .Ltmp0+2
+; CHECK: lhi %r2, 0
+; CHECK: br %r14
+entry:
+  %ptr = getelementptr i32, i32 *%base, i64 %offset
+  %b = load i32, i32 *%ptr
+  %cmp = icmp ugt i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 0
+}
+
+; Verify that we don't attempt to use the compare and trap grande
+; instruction with an index operand.
+define i64 @f4(i64 %a, i64 *%base, i64 %offset) {
+; CHECK-LABEL: f4:
+; CHECK: clg %r2, 0(%r{{[0-5]}},%r3)
+; CHECK-LABEL: .Ltmp1
+; CHECK: jh .Ltmp1+2
+; CHECK: lghi %r2, 0
+; CHECK: br %r14
+entry:
+  %ptr = getelementptr i64, i64 *%base, i64 %offset
+  %b = load i64, i64 *%ptr
+  %cmp = icmp ugt i64 %a, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 0
+}
+
diff --git a/test/CodeGen/SystemZ/trap-03.ll b/test/CodeGen/SystemZ/trap-03.ll
new file mode 100644
index 000000000000..d3cc882add75
--- /dev/null
+++ b/test/CodeGen/SystemZ/trap-03.ll
@@ -0,0 +1,157 @@
+; Test load-and-trap instructions (LAT/LGAT)
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 | FileCheck %s
+
+declare void @llvm.trap()
+
+; Check LAT with no displacement.
+define i32 @f1(i32 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: lat %r2, 0(%r2)
+; CHECK: br %r14
+entry:
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 %val
+}
+
+; Check the high end of the LAT range.
+define i32 @f2(i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: lat %r2, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 %val
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i32 @f3(i32 *%src) {
+; CHECK-LABEL: f3:
+; CHECK: agfi %r2, 524288
+; CHECK: lat %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 %val
+}
+
+; Check that LAT allows an index.
+define i32 @f4(i64 %src, i64 %index) {
+; CHECK-LABEL: f4:
+; CHECK: lat %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 %val
+}
+
+; Check LGAT with no displacement.
+define i64 @f5(i64 *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: lgat %r2, 0(%r2)
+; CHECK: br %r14
+entry:
+  %val = load i64, i64 *%ptr
+  %cmp = icmp eq i64 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %val
+}
+
+; Check the high end of the LGAT range.
+define i64 @f6(i64 *%src) {
+; CHECK-LABEL: f6:
+; CHECK: lgat %r2, 524280(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 65535
+  %val = load i64, i64 *%ptr
+  %cmp = icmp eq i64 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %val
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f7(i64 *%src) {
+; CHECK-LABEL: f7:
+; CHECK: agfi %r2, 524288
+; CHECK: lgat %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i64, i64 *%src, i64 65536
+  %val = load i64, i64 *%ptr
+  %cmp = icmp eq i64 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %val
+}
+
+; Check that LGAT allows an index.
+define i64 @f8(i64 %src, i64 %index) {
+; CHECK-LABEL: f8:
+; CHECK: lgat %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i64 *
+  %val = load i64, i64 *%ptr
+  %cmp = icmp eq i64 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %val
+}
diff --git a/test/CodeGen/SystemZ/trap-04.ll b/test/CodeGen/SystemZ/trap-04.ll
new file mode 100644
index 000000000000..9e7897e6321a
--- /dev/null
+++ b/test/CodeGen/SystemZ/trap-04.ll
@@ -0,0 +1,170 @@
+; Test load-and-trap instructions (LLGFAT/LLGFTAT)
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 | FileCheck %s
+
+declare void @llvm.trap()
+
+; Check LLGFAT with no displacement.
+define i64 @f1(i32 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: llgfat %r2, 0(%r2)
+; CHECK: br %r14
+entry:
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %cmp = icmp eq i64 %ext, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %ext
+}
+
+; Check the high end of the LLGFAT range.
+define i64 @f2(i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: llgfat %r2, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %cmp = icmp eq i64 %ext, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %ext
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f3(i32 *%src) {
+; CHECK-LABEL: f3:
+; CHECK: agfi %r2, 524288
+; CHECK: llgfat %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %cmp = icmp eq i64 %ext, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %ext
+}
+
+; Check that LLGFAT allows an index.
+define i64 @f4(i64 %src, i64 %index) {
+; CHECK-LABEL: f4:
+; CHECK: llgfat %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %cmp = icmp eq i64 %ext, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %ext
+}
+
+; Check LLGTAT with no displacement.
+define i64 @f5(i32 *%ptr) {
+; CHECK-LABEL: f5:
+; CHECK: llgtat %r2, 0(%r2)
+; CHECK: br %r14
+entry:
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %and = and i64 %ext, 2147483647
+  %cmp = icmp eq i64 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %and
+}
+
+; Check the high end of the LLGTAT range.
+define i64 @f6(i32 *%src) {
+; CHECK-LABEL: f6:
+; CHECK: llgtat %r2, 524284(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %and = and i64 %ext, 2147483647
+  %cmp = icmp eq i64 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %and
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define i64 @f7(i32 *%src) {
+; CHECK-LABEL: f7:
+; CHECK: agfi %r2, 524288
+; CHECK: llgtat %r2, 0(%r2)
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %and = and i64 %ext, 2147483647
+  %cmp = icmp eq i64 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %and
+}
+
+; Check that LLGTAT allows an index.
+define i64 @f8(i64 %src, i64 %index) {
+; CHECK-LABEL: f8:
+; CHECK: llgtat %r2, 524287(%r3,%r2)
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %val = load i32, i32 *%ptr
+  %ext = zext i32 %val to i64
+  %and = and i64 %ext, 2147483647
+  %cmp = icmp eq i64 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i64 %and
+}
+
diff --git a/test/CodeGen/SystemZ/trap-05.ll b/test/CodeGen/SystemZ/trap-05.ll
new file mode 100644
index 000000000000..7aabedbfadd3
--- /dev/null
+++ b/test/CodeGen/SystemZ/trap-05.ll
@@ -0,0 +1,92 @@
+; Test load-and-trap instructions (LFHAT)
+; See comments in asm-18.ll about testing high-word operations.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 \
+; RUN:   -no-integrated-as | FileCheck %s
+
+declare void @llvm.trap()
+
+; Check LAT with no displacement.
+define void @f1(i32 *%ptr) {
+; CHECK-LABEL: f1:
+; CHECK: lfhat [[REG:%r[0-9]+]], 0(%r2)
+; CHECK: stepa [[REG]]
+; CHECK: br %r14
+entry:
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  call void asm sideeffect "stepa $0", "h"(i32 %val)
+  ret void;
+}
+
+; Check the high end of the LAT range.
+define void @f2(i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: lfhat [[REG:%r[0-9]+]], 524284(%r2)
+; CHECK: stepa [[REG]]
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131071
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  call void asm sideeffect "stepa $0", "h"(i32 %val)
+  ret void;
+}
+
+; Check the next word up, which needs separate address logic.
+; Other sequences besides this one would be OK.
+define void @f3(i32 *%src) {
+; CHECK-LABEL: f3:
+; CHECK: agfi %r2, 524288
+; CHECK: lfhat [[REG:%r[0-9]+]], 0(%r2)
+; CHECK: stepa [[REG]]
+; CHECK: br %r14
+  %ptr = getelementptr i32, i32 *%src, i64 131072
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  call void asm sideeffect "stepa $0", "h"(i32 %val)
+  ret void;
+}
+
+; Check that LAT allows an index.
+define void @f4(i64 %src, i64 %index) {
+; CHECK-LABEL: f4:
+; CHECK: lfhat [[REG:%r[0-9]+]], 524287(%r3,%r2)
+; CHECK: stepa [[REG]]
+; CHECK: br %r14
+  %add1 = add i64 %src, %index
+  %add2 = add i64 %add1, 524287
+  %ptr = inttoptr i64 %add2 to i32 *
+  %val = load i32, i32 *%ptr
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @llvm.trap()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  call void asm sideeffect "stepa $0", "h"(i32 %val)
+  ret void;
+}
+
diff --git a/test/CodeGen/SystemZ/vec-args-06.ll b/test/CodeGen/SystemZ/vec-args-06.ll
index b26131ca1d4e..e8b59c697fd5 100644
--- a/test/CodeGen/SystemZ/vec-args-06.ll
+++ b/test/CodeGen/SystemZ/vec-args-06.ll
@@ -42,29 +42,29 @@ define { <2 x double>, <2 x double>, <2 x double>, <2 x double>,
 ; CHECK-LABEL: f2:
 ; CHECK: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 128(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 128(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 112(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 112(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 96(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 96(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 80(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 80(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 64(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 64(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 48(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 48(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 32(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 32(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
-; CHECK: vst [[VTMP]], 16(%r2)
-; CHECK: larl [[TMP:%r[0-5]]], .LCPI
+; CHECK-DAG: vst [[VTMP]], 16(%r2)
+; CHECK-DAG: larl [[TMP:%r[0-5]]], .LCPI
 ; CHECK: vl [[VTMP:%v[0-9]+]], 0([[TMP]])
 ; CHECK: vst [[VTMP]], 0(%r2)
 ; CHECK: br %r14
diff --git a/test/CodeGen/SystemZ/vec-perm-12.ll b/test/CodeGen/SystemZ/vec-perm-12.ll
index b70b13d90682..c6af5d051b87 100644
--- a/test/CodeGen/SystemZ/vec-perm-12.ll
+++ b/test/CodeGen/SystemZ/vec-perm-12.ll
@@ -7,9 +7,9 @@
 
 define <4 x i32> @f1(<4 x i32> %x, i64 %y) {
 ; CHECK-CODE-LABEL: f1:
-; CHECK-CODE: vlvgf [[ELT:%v[0-9]+]], %r2, 0
-; CHECK-CODE: larl [[REG:%r[0-5]]],
-; CHECK-CODE: vl [[MASK:%v[0-9]+]], 0([[REG]])
+; CHECK-CODE-DAG: vlvgf [[ELT:%v[0-9]+]], %r2, 0
+; CHECK-CODE-DAG: larl [[REG:%r[0-5]]],
+; CHECK-CODE-DAG: vl [[MASK:%v[0-9]+]], 0([[REG]])
 ; CHECK-CODE: vperm %v24, %v24, [[ELT]], [[MASK]]
 ; CHECK-CODE: br %r14
 
diff --git a/test/CodeGen/SystemZ/vec-perm-13.ll b/test/CodeGen/SystemZ/vec-perm-13.ll
index 708d8de53f86..d4aeffc66417 100644
--- a/test/CodeGen/SystemZ/vec-perm-13.ll
+++ b/test/CodeGen/SystemZ/vec-perm-13.ll
@@ -19,8 +19,8 @@ define <4 x i16> @f1(<4 x i16> %x) {
 ; CHECK-VECTOR-NEXT: .space  1                                        
 ; CHECK-VECTOR-NEXT: .byte   6
 ; CHECK-VECTOR-NEXT: .byte   7
-; CHECK-VECTOR-NEXT: .byte   16
-; CHECK-VECTOR-NEXT: .byte   17
+; CHECK-VECTOR-NEXT: .byte   22
+; CHECK-VECTOR-NEXT: .byte   23
 ; CHECK-VECTOR-NEXT: .space  1                                        
 ; CHECK-VECTOR-NEXT: .space  1                                        
 ; CHECK-VECTOR-NEXT: .space  1